Upgrade to Pro — share decks privately, control downloads, hide ads and more …

勾配流から乖離した学習ダイナミクス

Avatar for Han Bao Han Bao
September 10, 2026

 勾配流から乖離した学習ダイナミクス

数理最適化で繋ぐ幾何解析と応用数学 第1回( https://sites.google.com/g.ecc.u-tokyo.ac.jp/26h01996/%E7%AC%AC1%E5%9B%9E )の連続講演で用いた資料です。

Avatar for Han Bao

Han Bao

September 10, 2026

More Decks by Han Bao

Other Decks in Science

Transcript

  1. む 2022 2022 2025 2025 2025 工 白 力 大

    大 : 大 む JST BOOST 士 2 / BAO Han
  2. 3 n X <latexit sha1_base64="DYIBXtwxLOKk3ENxjG8Y7VSVr1w=">AAACjXicfVBdTxNBFJ0ufmBRKfDoy8SGBI02uwSBBzAYTPTFiMYCCdNuZqd3y4T52MzcBZrN/h5/ja8a/42zpSZajDeZ5OTcc3PmnKxQ0mMc/2xFC3fu3ru/+KC99PDR4+XOyuqxt6UT0BdWWXeacQ9KGuijRAWnhQOuMwUn2cVhsz+5BOelNV9wUsBA87GRuRQcA5V23jAtTVpdMWmY5nieZdXnejiqKcsdF4lhvtRpJfeTemjoxiSV9CW9GjK0Bb1O5TM63Ew73bgXT4feBskMdMlsjtKV1hYbWVFqMCgU9/4siQscVNyhFArqNis9FFxc8DGcBWi4Bj+opllruh6YEc2tC88gnbJ/XlRcez/RWVA2efz8riH/ucv0nDPmu4NKmqJEMOLGOC8VRUubIulIOhCoJgFw4WT4OxXnPJSGoe52m72FEM7Bh2D0sQDH0brnFeNuHAqvQ9gxe9Gg/wn59W9hQKHnZL7V2+B4s5ds97Y/bXUPDmeNL5In5CnZIAnZIQfkPTkifSLIV/KNfCc/ouXoVbQXvb6RRq3ZzRr5a6J3vwA6kck7</latexit> 1 min (yi d w2R

    n i=1 > w xi ) <latexit sha1_base64="S+Gu7cjvBnYt/ScT9ZZVE+BdpiU=">AAACl3icfVBdaxQxFM2OX3X96seT+BJcBBFdZpZSfWuhKoVSbMFtC812uJO9MxuaZEKSsV2m86P8NeKb/hMzuyvoVrwQOJxzbu69JzNSOB/H3zvRrdt37t5bud998PDR4yera+vHrqwsxyEvZWlPM3AohcahF17iqbEIKpN4kl3stvrJF7ROlPqznxocKSi0yAUHH6h0dZ8Jnac1m4Cnlw1lrjJp3QIFfpJl9YeGXc9F+oZesut0cD6gDNxUmWAWhYLzAcstcDoOv/XifjwrehMkC9AjizpM1zqbbFzySqH2XIJzZ0ls/KgG6wWX2HRZ5dAAv4ACzwLUoNCN6tnVDX0RmDHNSxue9nTG/tlRg3Jhzyw422PcstaS/9QytTTZ5+9GtdCm8qj5fHBeSepL2kZKx8Ii93IaAHArwu6UTyBk4kPw3S57j+E4iwdh0CeDFnxpX9UMbKGEbsKxBXvdov8Z4eq3MaCQc7Kc6k1wPOgnW/2to83ezu4i8RXyjDwnL0lC3pIdskcOyZBw8pV8Iz/Iz+hptB19jPbm1qiz6Nkgf1V09Asi582O</latexit> <latexit sha1_base64="4RjtynnJq4DEhuEWy3woq+SfFWU=">AAACbXicfZDfahNBFMYnq9UaraaKV4qMhtI/aNiVUr0RCvXCG2kLTRvopuHs5CQZMn+WmbNqWHLt03jbPotP0VfoJN2CpuKBgd985xvOnC/LlfQUx79r0Z27S/fuLz+oP3y08vhJY/XpsbeFE9gWVlnXycCjkgbbJElhJ3cIOlN4ko33Zv2Tb+i8tOaIJjl2NQyNHEgBFKRe43U6AuLf+Se+0TlLyea8s3lWvkumvLpOeo1m3IrnxW9DUkGTVXXQW61tp30rCo2GhALvT5M4p24JjqRQOK2nhcccxBiGeBrQgEbfLee7TPlaUPp8YF04hvhc/fNFCdr7ic6CUwON/GJvJv6zl+mFyTT42C2lyQtCI64HDwrFyfJZULwvHQpSkwAgnAx/52IEDgSFOOv19DOG5Rx+DYP2c3RA1m2VKbihlmYalh2mb2f0PyP8uDEGCjkni6nehuP3rWSntXO43dzdqxJfZi/YG7bBEvaB7bIv7IC1mWA/2S92zi5ql9Hz6GX06toa1ao3z9hfFa1fAbBjvBs=</latexit> > ŵ = (X X) 1 > X y inf sup Ekŵ ŵ w 2 d 2 2 wk2 ⇣ n
  3. n X 1 min (yi f 2F n i=1 f

    (xi )) <latexit sha1_base64="2x7MhXGR2b/34jtXMenoIhjGETY=">AAAClnicfZBdaxNBFIYn61eNX6neCN4MBmFTNd2VUr2SQhVFEGMxbaETw+zs2c3Q+Vhnzoph2f/krxG80p/iJI2gqXhg4Jn3vMOZ82aVkh6T5HsnunDx0uUrG1e7167fuHmrt3n70NvaCRgLq6w7zrgHJQ2MUaKC48oB15mCo+x0f9E/+gzOS2s+4LyCiealkYUUHIM07b05iNmMIy0G9DE9iIuPWwPKFHjvpabMf3LYsMJx0eRBtiWNzXY+oA+XlzjdZjko5IO2MW077fWTYbIseh7SFfTJqkbTzc4Oy62oNRgUint/kiYVThruUAoFbZfVHiouTnkJJwEN1+AnzXLplj4ISk4L68IxSJfqny8arr2f6yw4NceZX+8txH/2Mr02GYtnk0aaqkYw4mxwUSuKli4Spbl0IFDNA3DhZPg7FTMeMsOQe7fLXkBYzsHbMOhdBY6jdVsN467U0rRh2ZI9WtD/jPzLb2OgkHO6nup5OHwyTHeHu+93+nv7q8Q3yD1yn8QkJU/JHnlNRmRMBPlKvpEf5Gd0N3oevYxenVmjzurNHfJXRaNfomnKjQ==</latexit> ˆ R(f ) R(f ) . ⇤ f⇤ ⇤ <latexit sha1_base64="ps4yf7h8CyqgsN84qz2RaOn5wNU=">AAACTXicfZBNSwMxEIaz9aNav6oevSwWQUTKrpTqUdCDF1HRtoJbZTadXUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHKmTae9+IUJianposzs6W5+YXFpfLySlMnmaLYoAlP1GUIGjmT2DDMcLxMFYIIObbC7sGg37pDpVkiL0wvxbaAWLKIUTBWOo+ut27KFa/qDcv9Df4IKmRUpzfLTi3oJDQTKA3loPWV76WmnYMyjHLsl4JMYwq0CzFeWZQgULfz4a59d8MqHTdKlH3SuEP1+48chNY9EVqnAHOrx3sD8c9eKMYmm2ivnTOZZgYl/RwcZdw1iTsIwu0whdTwngWgitndXXoLCqixcZVKwSHa4xQe20EnKSowidrKA1CxYLJvj42D7QH9Z4T7L6Mlm7M/nupvaO5U/Xq1flar7B+MEp8ha2SdbBKf7JJ9ckROSYNQEpMH8kienGfn1Xlz3j+tBWf0Z5X8qELxA//Zs/w=</latexit> 見 f <latexit sha1_base64="ps4yf7h8CyqgsN84qz2RaOn5wNU=">AAACTXicfZBNSwMxEIaz9aNav6oevSwWQUTKrpTqUdCDF1HRtoJbZTadXUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHKmTae9+IUJianposzs6W5+YXFpfLySlMnmaLYoAlP1GUIGjmT2DDMcLxMFYIIObbC7sGg37pDpVkiL0wvxbaAWLKIUTBWOo+ut27KFa/qDcv9Df4IKmRUpzfLTi3oJDQTKA3loPWV76WmnYMyjHLsl4JMYwq0CzFeWZQgULfz4a59d8MqHTdKlH3SuEP1+48chNY9EVqnAHOrx3sD8c9eKMYmm2ivnTOZZgYl/RwcZdw1iTsIwu0whdTwngWgitndXXoLCqixcZVKwSHa4xQe20EnKSowidrKA1CxYLJvj42D7QH9Z4T7L6Mlm7M/nupvaO5U/Xq1flar7B+MEp8ha2SdbBKf7JJ9ckROSYNQEpMH8kienGfn1Xlz3j+tBWf0Z5X8qELxA//Zs/w=</latexit> 一 4 <latexit sha1_base64="zd0qs9YnZx3rv+u8W9SoszuY7LY=">AAACh3icfVBdSxtBFJ2s/dC01WgffRkahCg27oqkghQUS+lLUaFRwYnL3cndODgzu8zMimHZH9Nf09f20X/jbEyhRumFgcO553LmnCSXwrowvGsEcy9evno9v9B88/bd4lJreeXUZoXh2OeZzMx5Ahal0Nh3wkk8zw2CSiSeJdeH9f7sBo0Vmf7hxjkOFIy0SAUH56m4tceU0HGZMqGZAnfFQdKvFWWpAR5pZgsVl+JzVF1q2hnHgn6kaec2Fuvr9HI7brXDbjgZ+hREU9Am0zmOlxs7bJjxQqF2XIK1F1GYu0EJxgkusWqywmIO/BpGeOGhBoV2UE5SVnTNM0OaZsY/7eiE/feiBGXtWCVeWSexs7uafHaXqBlnl+4OSqHzwqHmD8ZpIanLaF0hHQqD3MmxB8CN8H+n/Ap8Yc4X3WyyL+jDGfzujY5yNOAys1EyMCNfdeXDjthmjf4nhNu/Qo98z9Fsq0/B6XY36nV7Jzvt/cNp4/NklXwgHRKRT2SffCPHpE84+Ul+kd/kT7AQbAW9YPdBGjSmN+/JowkO7gGTOMXH</latexit> r 2 d log(n/d) + log(1/ ) n (cf. )
  4. 5 x <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y=W y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit>

    3 (W 2 1 (W x))
  5. 6 x <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y=W y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit>

    3 (W 2 1 (W x))
  6. 7 x <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y=W y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit>

    3 (W 2 1 (W x))
  7. 8 x <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 1 E(y, ytrue ) = (y

    2 <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> 3 Wij 3 Wij @E 3 @Wij <latexit sha1_base64="UYf6vVaAUMM2bux1oNQvUSZOrcQ=">AAACjXicfZDfShwxFMazY612rXXVy96ELkIpssz4/0JE0aI3UoWuKzjrciZ7Zo0mmSHJiMMwz9On6a3i2zSzrlTX0gOBH993kpPzRangxvr+Y82beDf5fmr6Q33m4+ynucb8wplJMs2wzRKR6PMIDAqusG25FXieagQZCexEN/uV37lFbXiifto8xa6EgeIxZ2Cd1GvshbEGVoQpaMtB0O/lX87LMTN/YXZ6Bb8uL1fLXqPpt/xh0bcQjKBJRnXSm6+thf2EZRKVZQKMuQj81HaL6mEmsKyHmcEU2A0M8MKhAommWwx3LemSU/o0TrQ7ytKh+vJGAdKYXEauU4K9MuNeJf7Ti+TYZBtvdQuu0syiYk+D40xQm9AqSNrnGpkVuQNgmru/U3YFLi/r4q7XwwN0y2k8doN+pKjBJvpbEYIeSK5Kt+wgXK7of41w99zoyOUcjKf6Fs5WWsFGa+N0rbm7P0p8mnwmX8hXEpBNskuOyAlpE0Z+kd/knjx4c966t+3tPLV6tdGdRfKqvMM/f/3LbA==</latexit> <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y=W 3 (W 2 1 (W x)) @E @y @y @Wij3 ytrue ) 2
  8. 9 x <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 1 E(y, ytrue ) = (y

    2 <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> 3 Wij @E 3 @Wij 2 Wij @E 2 @Wij <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> 3 Wij <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> 2 Wij X @E @y @W 3 kl 3 W2 @y @Wkl ij <latexit sha1_base64="SdUMqO14fIfUv258zoU2mi9fPy0=">AAACuXicfVHRShtBFJ1dW5vG1ib2sS9DgyBFwq4VK/giWKEvpRaMEdwY7k7uJmNmZpeZ2dJl2a/0yU/xzdmYtnYVLwwczjmXM3MmzgQ3NghuPH/lxcvVV63X7bU3b9ffdbobZybNNcMBS0Wqz2MwKLjCgeVW4HmmEWQscBjPj2p9+Au14ak6tUWGIwlTxRPOwDpq3JGRyeW4nG+LKko0sDLKQFsOgh5X/3DRFIsH4tCti+ryc9Pzly8d4lfV5U417vSCfrAY+hiES9AjyzkZd73daJKyXKKyTIAxF2GQ2VFZRzCBVTvKDWbA5jDFCwcVSDSjctFLRTcdM6FJqt1Rli7YhxslSGMKGTunBDszTa0mn9Ri2Ui2yf6o5CrLLSp2H5zkgtqU1qXTCdfIrCgcAKa5uztlM3BtWfc17Xb0Fd3jNH53QT8y1GBT/amMQE8lV5V77DTartFzRvj9x+iQ6zlstvoYnO30w73+3s/d3uHRsvEW+UA+ki0Ski/kkHwjJ2RAGLkmt57vrfgHPvgz/+re6nvLnffkv/HNHf6u3K0=</latexit> <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y=W 3 (W 2 1 (W x)) ytrue ) k,l 2
  9. 1 E(y, ytrue ) = (y 2 <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> y

    <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> 3 Wij @E 3 @Wij 2 Wij @E 2 @Wij 1 Wij @E 1 @Wij <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> 3 Wij <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> 2 Wij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y=W 0 1 x <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 3 (W 2 1 (W x)) 1 Wij ytrue ) 2
  10. 3 Wij @E 3 @Wij 2 Wij @E 2 @Wij

    1 Wij @E 1 @Wij <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> <latexit sha1_base64="16tW0XfXb2RwDG2mlIPN3uCIsSE=">AAACS3icfZDbSgMxEIaz1Xqop6qX3iwWQUTKrkj1UlDBG7EFe4BuKbPptAaT7JJkxbL0CbzVx/IBfA7vxAvTg6CtOBD4+OcPM/OHMWfaeN6bk5mbzy4sLi3nVlbX1jfym1s1HSWKYpVGPFKNEDRyJrFqmOHYiBWCCDnWw/vzYb/+gEqzSN6afowtAT3JuoyCsVLlsp0veEVvVO4s+BMokEmV25vOcdCJaCJQGspB66bvxaaVgjKMchzkgkRjDPQeeti0KEGgbqWjTQfunlU6bjdS9knjjtSfP1IQWvdFaJ0CzJ2e7g3FP3uhmJpsuqetlMk4MSjpeHA34a6J3GEMbocppIb3LQBVzO7u0jtQQI0NK5cLLtAep/DaDrqJUYGJ1EEagOoJJgf22F5wOKT/jPD4bbRkc/anU52F2lHRLxVLlePC2fkk8SWyQ3bJPvHJCTkjV6RMqoQSJE/kmbw4r8678+F8jq0ZZ/Jnm/yqTPYLcpGzPw==</latexit> E 3 Wij <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> 2 Wij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> 1 Wij <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> 1 1 1 E(y, ytrue ) = (y 2 <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> dW = dt rE(W ) ytrue ) 2
  11. む <latexit sha1_base64="dBuiNwrvo/kgc9MCDMIOd0lY2Ec=">AAACXHicfZDfShtBFMZnV2ttrJooeNOboaGgUsJuEdubgpBeeFOq0BjBDcvZyUkcMjO7zJwNDUvexFt9p970WTr5I2gsPTDw4zvfcM75skJJR1H0OwjX1l9tvN58U9t6u72zW2/sXbm8tAI7Ile5vc7AoZIGOyRJ4XVhEXSmsJuN2rN+d4zWydz8pEmBPQ1DIwdSAHkprde7h6MECY74V95Nq9E0rTejVjQv/hLiJTTZsi7SRnCS9HNRajQkFDh3E0cF9SqwJIXCaS0pHRYgRjDEG48GNLpeNV99yj94pc8HufXPEJ+rT39UoJ2b6Mw7NdCtW+3NxH/2Mr0ymQZfepU0RUloxGLwoFSccj7LhfelRUFq4gGElX53Lm7BgiCfXq2WfEN/nMXvftCPAi1Qbo+rBOxQSzP1xw6TjzP6nxF+PRo9+Zzj1VRfwtWnVnzaOr08aZ61l4lvsnfsPTtkMfvMztg5u2AdJtiY3bF79hD8CdfDrXB7YQ2D5Z999qzCg79bc7YD</latexit> <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW = dt W (k⌘)

    = Wk <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit> Wk+1 = Wk rE(W ) ⌘rE(Wk ) <latexit sha1_base64="IfFSNFS4MAtiGLpHp//LggCLChM=">AAACeXicfZDfahNBFMYn22rb+KepXvZmNAhJNWG3lupNoVAFb4oVTFPohnB2cjYdOjO7zJwthmWfok/jrX2KPos3zqYRNBUPDPz4znc4c74kV9JRGN42gpXVBw/X1jeajx4/ebrZ2np26rLCChyITGX2LAGHShockCSFZ7lF0InCYXJ5VPeHV2idzMxXmuU40jA1MpUCyEvjVi9OLYhy2KHXMRJ0e566VVlzxQ94LzaQKOAfO7XeHbfaYT+cF78P0QLabFEn463GXjzJRKHRkFDg3HkU5jQqwZIUCqtmXDjMQVzCFM89GtDoRuX8roq/8sqEp5n1zxCfq39OlKCdm+nEOzXQhVvu1eI/e4le2kzp+1EpTV4QGnG3OC0Up4zXofGJtChIzTyAsNL/nYsL8MGRj7bZjD+gP87isV/0OUcLlNmdMgY71dJU/thp/Kam/xnh22+jJ59ztJzqfTjd7Uf7/f0ve+3Do0Xi62ybvWQdFrF37JB9YidswAS7Zt/ZD3bT+Bm8CDrBzp01aCxmnrO/Knj7C1wTwGA=</latexit> 力 力 W (t + ⌘) ⌘ 2 1 む W (t) = rE(W (t))
  12. む <latexit sha1_base64="dBuiNwrvo/kgc9MCDMIOd0lY2Ec=">AAACXHicfZDfShtBFMZnV2ttrJooeNOboaGgUsJuEdubgpBeeFOq0BjBDcvZyUkcMjO7zJwNDUvexFt9p970WTr5I2gsPTDw4zvfcM75skJJR1H0OwjX1l9tvN58U9t6u72zW2/sXbm8tAI7Ile5vc7AoZIGOyRJ4XVhEXSmsJuN2rN+d4zWydz8pEmBPQ1DIwdSAHkprde7h6MECY74V95Nq9E0rTejVjQv/hLiJTTZsi7SRnCS9HNRajQkFDh3E0cF9SqwJIXCaS0pHRYgRjDEG48GNLpeNV99yj94pc8HufXPEJ+rT39UoJ2b6Mw7NdCtW+3NxH/2Mr0ymQZfepU0RUloxGLwoFSccj7LhfelRUFq4gGElX53Lm7BgiCfXq2WfEN/nMXvftCPAi1Qbo+rBOxQSzP1xw6TjzP6nxF+PRo9+Zzj1VRfwtWnVnzaOr08aZ61l4lvsnfsPTtkMfvMztg5u2AdJtiY3bF79hD8CdfDrXB7YQ2D5Z999qzCg79bc7YD</latexit> <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW = dt W (k⌘)

    = Wk <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit> Wk+1 = Wk rE(W ) ⌘rE(Wk ) <latexit sha1_base64="JAD/0sEwNQXS374OIM3uu9xGAOs=">AAACaHicfZDPShxBEMZ7xySazb/VHIIkh8ElYEJYZkTUo2AOuYgGsq7gLEtNT83a2P/srhGXYS55mlzj2/gKeYr0riska0hBw4+vvqK6vtxK4SlJblvR0qPHT5ZXnrafPX/x8lVnde3Em8px7HMjjTvNwaMUGvskSOKpdQgqlzjILw6m/cEVOi+M/kYTi0MFYy1KwYGCNOq8y0oHvC4Gm/ShqQtqsssKigysdeZ61OkmvWRW8UNI59Bl8zoerba2s8LwSqEmLsH7szSxNKzBkeASm3ZWebTAL2CMZwE1KPTDenZGE78PShGXxoWnKZ6pf07UoLyfqDw4FdC5X+xNxX/2crWwmcq9YS20rQg1v1tcVjImE08zigvhkJOcBADuRPh7zM8h5EQhyXY7+4zhOIeHYdGRRQdk3Mc6AzdWQjfh2HH2aUr/M8L1vTFQyDldTPUhnGz10p3eztft7v7BPPEV9pZtsE2Wsl22z76wY9ZnnH1nP9hPdtP6FXWiN9H6nTVqzWdes78q2vgN80K8eQ==</latexit> 力 力 <latexit sha1_base64="IfFSNFS4MAtiGLpHp//LggCLChM=">AAACeXicfZDfahNBFMYn22rb+KepXvZmNAhJNWG3lupNoVAFb4oVTFPohnB2cjYdOjO7zJwthmWfok/jrX2KPos3zqYRNBUPDPz4znc4c74kV9JRGN42gpXVBw/X1jeajx4/ebrZ2np26rLCChyITGX2LAGHShockCSFZ7lF0InCYXJ5VPeHV2idzMxXmuU40jA1MpUCyEvjVi9OLYhy2KHXMRJ0e566VVlzxQ94LzaQKOAfO7XeHbfaYT+cF78P0QLabFEn463GXjzJRKHRkFDg3HkU5jQqwZIUCqtmXDjMQVzCFM89GtDoRuX8roq/8sqEp5n1zxCfq39OlKCdm+nEOzXQhVvu1eI/e4le2kzp+1EpTV4QGnG3OC0Up4zXofGJtChIzTyAsNL/nYsL8MGRj7bZjD+gP87isV/0OUcLlNmdMgY71dJU/thp/Kam/xnh22+jJ59ztJzqfTjd7Uf7/f0ve+3Do0Xi62ybvWQdFrF37JB9YidswAS7Zt/ZD3bT+Bm8CDrBzp01aCxmnrO/Knj7C1wTwGA=</latexit> 3 1 む dW W (t + ⌘) (t)W (t) ⇡= ⌘ dt rE(W (t))
  13. 1 min (yi f 2F n i=1 む f (xi

    )) む 目 力 ・ 見 目 一 高 む 5 非 1 n X <latexit sha1_base64="zd0qs9YnZx3rv+u8W9SoszuY7LY=">AAACh3icfVBdSxtBFJ2s/dC01WgffRkahCg27oqkghQUS+lLUaFRwYnL3cndODgzu8zMimHZH9Nf09f20X/jbEyhRumFgcO553LmnCSXwrowvGsEcy9evno9v9B88/bd4lJreeXUZoXh2OeZzMx5Ahal0Nh3wkk8zw2CSiSeJdeH9f7sBo0Vmf7hxjkOFIy0SAUH56m4tceU0HGZMqGZAnfFQdKvFWWpAR5pZgsVl+JzVF1q2hnHgn6kaec2Fuvr9HI7brXDbjgZ+hREU9Am0zmOlxs7bJjxQqF2XIK1F1GYu0EJxgkusWqywmIO/BpGeOGhBoV2UE5SVnTNM0OaZsY/7eiE/feiBGXtWCVeWSexs7uafHaXqBlnl+4OSqHzwqHmD8ZpIanLaF0hHQqD3MmxB8CN8H+n/Ap8Yc4X3WyyL+jDGfzujY5yNOAys1EyMCNfdeXDjthmjf4nhNu/Qo98z9Fsq0/B6XY36nV7Jzvt/cNp4/NklXwgHRKRT2SffCPHpE84+Ul+kd/kT7AQbAW9YPdBGjSmN+/JowkO7gGTOMXH</latexit> 2
  14. 1 min (yi f 2F n i=1 む f (xi

    )) む 目 力 ・ ・ 見 目 一 高 む 6 非 1 n X <latexit sha1_base64="zd0qs9YnZx3rv+u8W9SoszuY7LY=">AAACh3icfVBdSxtBFJ2s/dC01WgffRkahCg27oqkghQUS+lLUaFRwYnL3cndODgzu8zMimHZH9Nf09f20X/jbEyhRumFgcO553LmnCSXwrowvGsEcy9evno9v9B88/bd4lJreeXUZoXh2OeZzMx5Ahal0Nh3wkk8zw2CSiSeJdeH9f7sBo0Vmf7hxjkOFIy0SAUH56m4tceU0HGZMqGZAnfFQdKvFWWpAR5pZgsVl+JzVF1q2hnHgn6kaec2Fuvr9HI7brXDbjgZ+hREU9Am0zmOlxs7bJjxQqF2XIK1F1GYu0EJxgkusWqywmIO/BpGeOGhBoV2UE5SVnTNM0OaZsY/7eiE/feiBGXtWCVeWSexs7uafHaXqBlnl+4OSqHzwqHmD8ZpIanLaF0hHQqD3MmxB8CN8H+n/Ap8Yc4X3WyyL+jDGfzujY5yNOAys1EyMCNfdeXDjthmjf4nhNu/Qo98z9Fsq0/B6XY36nV7Jzvt/cNp4/NklXwgHRKRT2SffCPHpE84+Ul+kd/kT7AQbAW9YPdBGjSmN+/JowkO7gGTOMXH</latexit> 2
  15. W2 W3 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit>

    <latexit sha1_base64="9A8DVNCoJep5zMhNCyALJJT1r74=">AAACjHicfZBvaxNBEMY3578atU31pW8Wg5CIhrsa2oIUAhXxRcUKpin00jC3mSRL98+xOycNR76On8a3Cn4bN8kJmooDCz+eZ4bZebJcSU9x/LMW3bp95+69rfv1Bw8fbe80dh+feVs4gX1hlXXnGXhU0mCfJCk8zx2CzhQOsqvjpT/4gs5Laz7TPMehhqmREymAgjRq9Ob8iA8uT9J8JluDy/LkVbJYcSrGlnyQXlfWXquC5Lodau2326NGM+7Eq+I3Iamgyao6He3WuunYikKjIaHA+4skzmlYgiMpFC7qaeExB3EFU7wIaECjH5arUxf8eVDGfGJdeIb4Sv1zogTt/VxnoVMDzfymtxT/6WV6YzNNDoelNHlBaMR68aRQnCxf5sjH0qEgNQ8Awsnwdy5m4EBQSLteT99iOM7hh7DoY44OyLoXZQpuqqVZhGOn6csl/a8Rrn83Bgo5J5up3oSzvU6y39n/1G32jqvEt9hT9oy1WMIOWI+9Z6eszwT7yr6x7+xHtB11ozfR0bo1qlUzT9hfFb37BUc/xmw=</latexit> y=W 1 2 … x y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> L <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit> (W <latexit sha1_base64="6fpeLvjCHtT3PXJ9u3sazCj0s/E=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bIroh4FPXhQVLBWcKvMptMammSXJCuWpT/Cq/4sT/4Ub6a1glZxIPDwzhtm5k0ywY0NwzfPHxkdG5+YnCpNz8zOzS8sLl2ZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaBa0j7s9WsPpA1P1aXtZFSX2FK8yRlaJ9Vqt8XJVtS9WyiHlbBfwW+IBlCGQZ3fLXo7cSNluSRlmUBjbqIws/UCteVMULcU54YyZG1s0Y1DhZJMvejv2w3WnNIImql2T9mgr37/UaA0piMT55Ro781wryf+2Uvk0GTb3K8XXGW5JcU+BzdzEdg06IURNLgmZkXHATLN3e4Bu0eNzLrISqX4iNxxmk7doLOMNNpUbxQx6pbkquuObcWbPfrPiI9fRkcu52g41d9wtV2Jdiu7Fzvlg8NB4pOwAquwDhHswQEcwzlUgUEbnuAZXrxX790H3/+0+t7gzzL8KH/6AwZjs48=</latexit> <latexit sha1_base64="1jCUJB1wsCZJMZRmcibeh66EBBw=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi6horeDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpWJp+VrHqaJYpTGP1U0IGjmTWDXMcLxJFIIIOdbCzlG/X7tHpVksr0w3wbqASLIWo2CsdFm722kUy17FG5Q7Dv4QymRY542Ssxs0Y5oKlIZy0PrW9xJTz0AZRjn2CkGqMQHagQhvLUoQqOvZYNeeu26VptuKlX3SuAP1548MhNZdEVqnANPWo72++GcvFCOTTeugnjGZpAYl/RrcSrlrYrcfhNtkCqnhXQtAFbO7u7QNCqixcRUKwTHa4xSe2kFnCSowsdrMAlCRYLJnj42CrT79Z4SHb6Mlm7M/muo4XG9X/L3K3sVu+fBomPgMWSVrZIP4ZJ8ckhNyTqqEkog8kify7Lw4b8678/FlzTnDPyvkV+Xyn/RRs/Y=</latexit> L 1 W L 1 <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit> (· · · (W <latexit sha1_base64="xBJ4k0CUIqZOE77ZE/R1zKcuK5Q=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9CnrwoKhoreDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpeXiyrWOU0WxSmMeq5sQNHImsWqY4XiTKAQRcqyFncN+v3aPSrNYXplugnUBkWQtRsFY6bJ2d9JYLnllb1DuOPhDKJFhnTeKTiVoxjQVKA3loPWt7yWmnoEyjHLsFYJUYwK0AxHeWpQgUNezwa49d8MqTbcVK/ukcQfqzx8ZCK27IrROAaatR3t98c9eKEYmm9Z+PWMySQ1K+jW4lXLXxG4/CLfJFFLDuxaAKmZ3d2kbFFBj4yoUgiO0xyk8tYPOElRgYrWVBaAiwWTPHhsF2336zwgP30ZLNmd/NNVxuN4p+7vl3YtK6eBwmPgMWSPrZJP4ZI8ckGNyTqqEkog8kify7Lw4b8678/FlzTnDP6vkV+XynyPvtA8=</latexit> W L <latexit sha1_base64="Z9F6XKWw9ZrSkzsAWVfQaQfq1tU=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsSlGPhXrwIlawVnBLmU1n22CSXZKsWJb+Ba/6s7z6R7yJZmsFreJA4OGdN8zMGyacaeN5L05hZnZufqG4WFpaXlldWy9vXOk4VRRbNOaxug5BI2cSW4YZjteJQhAhx3Z428j77TtUmsXy0gwT7AjoSxYxCiaXgmTAuusVr+qNy/0N/gQqZFLNbtmpBb2YpgKloRy0vvG9xHQyUIZRjqNSkGpMgN5CH28sShCoO9l42ZG7Y5WeG8XKPmncsfr9RwZC66EIrVOAGejpXi7+2QvF1GQTHXcyJpPUoKSfg6OUuyZ28yTcHlNIDR9aAKqY3d2lA1BAjc2rVApO0B6n8MwOOk9QgYnVXhaA6gsmR/bYfrCf039GuP8yWrI5+9Op/oarg6p/WD28qFXqjUniRbJFtsku8ckRqZNT0iQtQsmAPJBH8uQ8O6/Om/P+aS04kz+b5EcVih9zXbS1</latexit> 3 2 1 (W ( (W x)))) · · · ))
  16. W <latexit sha1_base64="ijYar2CKkylYYvWA0xE4gjSVp9U=">AAACa3icfZDfShtBFMYnq602/Rf1rhU6NAiltGFXRb0RBHvhhaUWGldwk3B2chIH588yc1YMSy77NL3Vh+lD9B2cxBTaWHpg4Md3vsOZ8+WFkp7i+GctWlh89Hhp+Un96bPnL142VlZPvS2dwLawyrqzHDwqabBNkhSeFQ5B5wrT/PJw0k+v0HlpzTcaFdjRMDRyIAVQkHqNNyO+z9Pucdqtjj8m40z0LfkgbKXdzbSbXPcazbgVT4s/hGQGTTark95KbTvrW1FqNCQUeH+exAV1KnAkhcJxPSs9FiAuYYjnAQ1o9J1qesmYbwSlzwfWhWeIT9U/JyrQ3o90Hpwa6MLP9ybiP3u5nttMg71OJU1REhpxv3hQKk6WT2LifelQkBoFAOFk+DsXF+BAUAizXs8+YTjO4eew6EuBDsi691UGbqilGYdjh9mHCf3PCNe/jYFCzsl8qg/hdLOV7LR2vm43Dw5niS+z1+wte8cStssO2BE7YW0m2Hf2g92w29qvaC16Fa3fW6PabGaN/VXRxh0vz7tu</latexit> W2 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> 1 L

    y=W W 2 2 … x y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> L 1 W3 <latexit sha1_base64="6fpeLvjCHtT3PXJ9u3sazCj0s/E=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bIroh4FPXhQVLBWcKvMptMammSXJCuWpT/Cq/4sT/4Ub6a1glZxIPDwzhtm5k0ywY0NwzfPHxkdG5+YnCpNz8zOzS8sLl2ZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaBa0j7s9WsPpA1P1aXtZFSX2FK8yRlaJ9Vqt8XJVtS9WyiHlbBfwW+IBlCGQZ3fLXo7cSNluSRlmUBjbqIws/UCteVMULcU54YyZG1s0Y1DhZJMvejv2w3WnNIImql2T9mgr37/UaA0piMT55Ro781wryf+2Uvk0GTb3K8XXGW5JcU+BzdzEdg06IURNLgmZkXHATLN3e4Bu0eNzLrISqX4iNxxmk7doLOMNNpUbxQx6pbkquuObcWbPfrPiI9fRkcu52g41d9wtV2Jdiu7Fzvlg8NB4pOwAquwDhHswQEcwzlUgUEbnuAZXrxX790H3/+0+t7gzzL8KH/6AwZjs48=</latexit> <latexit sha1_base64="1jCUJB1wsCZJMZRmcibeh66EBBw=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi6horeDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpWJp+VrHqaJYpTGP1U0IGjmTWDXMcLxJFIIIOdbCzlG/X7tHpVksr0w3wbqASLIWo2CsdFm722kUy17FG5Q7Dv4QymRY542Ssxs0Y5oKlIZy0PrW9xJTz0AZRjn2CkGqMQHagQhvLUoQqOvZYNeeu26VptuKlX3SuAP1548MhNZdEVqnANPWo72++GcvFCOTTeugnjGZpAYl/RrcSrlrYrcfhNtkCqnhXQtAFbO7u7QNCqixcRUKwTHa4xSe2kFnCSowsdrMAlCRYLJnj42CrT79Z4SHb6Mlm7M/muo4XG9X/L3K3sVu+fBomPgMWSVrZIP4ZJ8ckhNyTqqEkog8kify7Lw4b8678/FlzTnDPyvkV+Xyn/RRs/Y=</latexit> 3 W 2 1 ···W W W x L 1 <latexit sha1_base64="xBJ4k0CUIqZOE77ZE/R1zKcuK5Q=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9CnrwoKhoreDWMptOt6FJdkmyYln6E7zqz/LsD/EmYloraCsOBB7eecPMvGHCmTae9+rkJianpvMzs4W5+YXFpeXiyrWOU0WxSmMeq5sQNHImsWqY4XiTKAQRcqyFncN+v3aPSrNYXplugnUBkWQtRsFY6bJ2d9JYLnllb1DuOPhDKJFhnTeKTiVoxjQVKA3loPWt7yWmnoEyjHLsFYJUYwK0AxHeWpQgUNezwa49d8MqTbcVK/ukcQfqzx8ZCK27IrROAaatR3t98c9eKEYmm9Z+PWMySQ1K+jW4lXLXxG4/CLfJFFLDuxaAKmZ3d2kbFFBj4yoUgiO0xyk8tYPOElRgYrWVBaAiwWTPHhsF2336zwgP30ZLNmd/NNVxuN4p+7vl3YtK6eBwmPgMWSPrZJP4ZI8ckGNyTqqEkog8kify7Lw4b8678/FlzTnDP6vkV+XynyPvtA8=</latexit> W L
  17. n X <latexit sha1_base64="tzzYQpnyTOWtYwrf/0n/Dr0ga2s=">AAACenicfZDPSiNBEMY7o7tq3F2je/TSGIT9G2Yk6F4WBD14ERU2RrDjUNOpiY3dPUN3z2IY5y18Gq/6Er6LBzsxwhoXCxp+fPUV1fUluRTWheF9LZiZffd+bn6hvvjh46elxvLKsc0Kw7HDM5mZkwQsSqGx44STeJIbBJVI7CYXO6N+9y8aKzL9xw1z7CkYaJEKDs5LcaPFlNBx2a0oSw3wSDNbqLgUv6PqTFN2NYwF/Um7l7FgV/RsI240w1Y4Lvoaogk0yaQO4+Vam/UzXijUjkuw9jQKc9crwTjBJVZ1VljMgV/AAE89alBoe+X4sIque6VP08z4px0dq/9OlKCsHarEOxW4czvdG4n/7SVqarNLf/VKofPCoeZPi9NCUpfRUWq0LwxyJ4cegBvh/075OfjAnM+2Xme76I8zuO8XHeRowGXmW8nADHy6lT92wH6M6C0jXD4bPfmco+lUX8PxRivabG0etZvbO5PE58kqWSNfSES2yDbZI4ekQzi5JjfkltzVHoK14Gvw/cka1CYzn8mLCtqPd1/CZQ==</latexit> n X <latexit sha1_base64="dclVvJvNWVZjlYam1xI1RZCRngA=">AAACmXicfZBdaxNBFIYnWz9q/ErrZW8GgyBiw04stRQKlXpRJMWKplvoZJfZydl06HwsM7OlYbu/yl/TW/0jTtIImooHBl7e8x7OnCcvpXA+jm9a0cq9+w8erj5qP37y9Nnzztr6iTOV5TDkRhp7mjMHUmgYeuElnJYWmMolJPnFwayfXIJ1wuhvflrCSLGJFoXgzAcr6xxRJXRWJ2lNdgdNg2lhGSeaukpltdgjTaoxvZ5mAm/iJB2E3GCTNJSPjXfBeJek/SQlV5mg1zjtZ51u3Ivnhe8KshBdtKjjbK21RceGVwq055I5d0bi0o9qZr3gEpo2rRyUjF+wCZwFqZkCN6rndzf4VXDGuDA2PO3x3P1zombKuanKQ1Ixf+6WezPzn71cLW32xc6oFrqsPGh+u7ioJPYGz6DisbDAvZwGwbgV4e+Yn7NA0gf07Tb9COE4C0dh0ecSLPPGvqkps5MAvwnHTujbmfpfkF39DgYVOJNlqnfFSb9HtnvbX7a6+wcL4qtoA71ErxFB79E+OkTHaIg4+o5u0A/0M9qIPkSH0afbaNRazLxAf1X09Rc14sz4</latexit> 1 min

    kyi W n i=1 1 min kyi W 1:L n i=1 2 W xi k L W W L 1 む NN NN ⾒ 1 2 1 1 2 (W , W ) ! 7 ( W , W ) ; e.g. homogeneity <latexit sha1_base64="ylwJ1qcXweHP6Doh9X58PmFiszI=">AAACgHicfVDRShtBFJ2s1tq0atRHX4aGQiwSd0Ow4pOgD76UWjBGcGO4O7mbDM7MLjN3xbDkR/o1fdU/6N90ElNao/TCwOGcc++de5JcSUdh+KsSLC2/WXm7+q76/sPa+kZtc+vSZYUV2BGZyuxVAg6VNNghSQqvcougE4Xd5PZkqnfv0DqZmQsa59jTMDQylQLIU/1au9G9ifZ496a1y2MNuaOMN2LlBwyAz6SYUgsi+su1dvu1etgMZ8VfgmgO6mxe5/3NSjseZKLQaEgocO46CnPqlWBJCoWTalw4zEHcwhCvPTSg0fXK2XkT/skzA55m1j9DfMb+21GCdm6sE+/UQCO3qE3JV7VEL2ym9LBXSpMXhEY8LU4LxX0m0+z4QFoUpMYegLDS/52LEfh0yCdcrcan6I+z+NUv+pajBcrs5zIGO9TSTPyxw3hviv5nhPs/Ro98ztFiqi/BZasZHTQPvrfrxyfzxFfZDvvIGixiX9gxO2PnrMME+8F+sgf2GARBI9gPoidrUJn3bLNnFRz9BhNdwYg=</latexit> 一 非 一 力 ⾒ 3 2 1 2 · · · W W W xi k NN = む 3 2 NN
  18. NN む 2 <latexit sha1_base64="4QfpGZq2Y1weFxAfMEeD1Z2f4gg=">AAACVXicfZDbSgMxEIaz67me9dKbYBFEpOyKqDeCoBfeiArWCt0qs+m0huawJFmxLH0Mb/WxxIcRTGsFreJA4OOfP8zMn2aCWxdFb0E4Nj4xOTU9U5qdm19YXFpeubY6NwyrTAttblKwKLjCquNO4E1mEGQqsJZ2jvv92gMay7W6ct0MGxLairc4A+elepce0trtTu02frxbKkeVaFD0N8RDKJNhXdwtB7tJU7NconJMgLX1OMpcowDjOBPYKyW5xQxYB9pY96hAom0Ug517dMMrTdrSxj/l6ED9/qMAaW1Xpt4pwd3b0V5f/LOXypHJrnXQKLjKcoeKfQ5u5YI6TfuB0CY3yJzoegBmuN+dsnswwJyPrVRKTtAfZ/DMDzrP0IDTZqtIwLQlVz1/bDvZ7tN/Rnj8MnryOcejqf6G651KvFfZu9wtHx0PE58ma2SdbJKY7JMjckouSJUwoskTeSYvwWvwHo6Hk5/WMBj+WSU/Klz8AFWWtJs=</latexit> [Saxe-McClelland-Ganguli y=W W x N2

    ⇥N1 1 W 2R 2 ,W 2 R N3 ⇥N2 <latexit sha1_base64="I2c8rPXgJiVsk1hZFoxoKID7eqQ=">AAACe3icfZDfShtBFMYna1tt+i/qZW8GQ8GKTXeD/24EQQveSC00ruAky9nJ2Tg4M7vMzIphzWP4NN7qQ/gwBScxhTaWHhj48Z3vcOZ8aSGFdWH4UAvmXrx8Nb/wuv7m7bv3HxqLSyc2Lw3HDs9lbk5TsCiFxo4TTuJpYRBUKjFOL/bH/fgSjRW5/umGBXYVDLTIBAfnpaTx9dtq/JnuUpYZ4FGb2VIlldiNRj3NroeJoF9o3GvHvegqEey6104azbAVToo+h2gKTTKt42SxtsH6OS8VasclWHsWhYXrVmCc4BJHdVZaLIBfwADPPGpQaLvV5LIR/eSVPs1y4592dKL+OVGBsnaoUu9U4M7tbG8s/rOXqpnNLtvpVkIXpUPNnxZnpaQup+PYaF8Y5E4OPQA3wv+d8nPwmTkfbr3ODtAfZ/DIL/peoAGXm7WKgRkooUf+2AFbH9P/jHD12+jJ5xzNpvocTtqtaKu19WOjubc/TXyBfCQrZJVEZJvskUNyTDqEkxtyS+7Ife1X0AzWgvUna1CbziyTvyrYfAR1KsFl</latexit> n {(xi , yi )}i=1 <latexit sha1_base64="A0WnP2aUlxWqCCxSnZ7ftO30sLE=">AAACYXicfZBNS8NAEIa38avWr7biyUuwCCpSEhH1Igh68CIq2CqYGibbaV3c3YTdjbSE/Biv+os8+0fcfghaxYGFh3feZWbeKOFMG897LzhT0zOzc8X50sLi0vJKuVJt6jhVFBs05rG6i0AjZxIbhhmOd4lCEBHH2+jpdNC/fUalWSxvTD/BloCuZB1GwVgpLK8F2VYvZLv9kG0HeZixYz9/sHrNq3vDcn+DP4YaGddVWCnsB+2YpgKloRy0vve9xLQyUIZRjnkpSDUmQJ+gi/cWJQjUrWy4f+5uWqXtdmJlnzTuUP3+IwOhdV9E1inAPOrJ3kD8sxeJicmmc9TKmExSg5KOBndS7prYHYTjtplCanjfAlDF7O4ufQQF1NgIS6XgDO1xCi/soMsEFZhY7WQBqK5gMrfHdoPdAf1nhN6X0ZLN2Z9M9Tc09+r+Qf3ger92cjpOvEjWyQbZIj45JCfknFyRBqEkIy/klbwVPpx5p+xUR1anMP6zSn6Us/4J0564ng==</latexit> む E(W ) = <latexit sha1_base64="rd5oajX48R65mKNFLX36m+8U7g4=">AAACl3icfVDbahRBEO0db3G9bcyT+NK4CCKyzGiIIgQDCRIQY6JuEkjvDj29tZMmfRm6a8IOw3xUvkZ80z+xZ7OCbsSCgsOpU5yqkxVKeozj753o2vUbN2+t3O7euXvv/oPe6sNDb0snYCissu444x6UNDBEiQqOCwdcZwqOsrPtdn50Ds5La75iVcBI89zIqRQcA5X2PrAvMtecvt2kDGGGHisFlPlSp7XcTJqxqVI5S+WYoS0ok4YyzfE0y+rPzbjeS18xlBo83UuTJu3140E8L3oVJAvQJ4vaT1c762xiRanBoFDc+5MkLnBUc4dSKGi6rPRQcHHGczgJ0PBgNarnXzf0aWAmdGpdaIN0zv65UXPtfaWzoGxP9suzlvznLNNLzjh9M6qlKUoEIy6Np6WiaGkbKZ1IBwJVFQAXTobbqTjljgsMwXe7bAfCcw4+BqNPBTiO1j2vGXe5lqYJz+bsRYv+J+Sz38KAQs7JcqpXweHLQbIx2DhY729tLxJfIY/JE/KMJOQ12SK7ZJ8MiSAX5Bv5QX5Gj6J30fto91IadRY7a+Svig5+AeYTzfs=</latexit> ⌃ := む <latexit sha1_base64="Rjxu2609D7TUSzr62u44TakxgT8=">AAACeHicfVBdSxtBFJ2s/bBprdE+9mVoKFWxYTeI9UUQ7IMvpRYbI7hJuDu5WQfnY5m5K4Ylf6K/xlf7L/pX+uQkplBj6YWBwznncuaerFDSUxz/qkVLT54+e778ov7y1crr1cba+qm3pRPYEVZZd5aBRyUNdkiSwrPCIehMYTe7PJzq3St0XlrzncYF9jTkRo6kAArUoLGdfkZFwLv9hO/zFAmqbr896adki430ROYa+MegtoNhc9Boxq14NvwxSOagyeZzPFir7aRDK0qNhoQC78+TuKBeBY6kUDipp6XHAsQl5HgeoAGNvlfNzprw94EZ8pF14RniM/bvjQq092OdBacGuvCL2pT8p5bphWQa7fUqaYqS0Ij74FGpOFk+7YwPpUNBahwACCfD37m4AAeCQrP1eqgwHOfwSwj6WqADsm6rSsHlWppJODZPt6fof0a4/mMMKPScLLb6GJy2W8lua/fbTvPgcN74MnvL3rENlrBP7IAdsWPWYYL9YDfslv2s/Y549CHavLdGtfnOG/ZgovYdeA+//w==</latexit> 1 W = ⌘W 2> Pn > N3 ⇥N1 y x 2 R ; i=1 i i 2 (⌃ 2 W W ) W = ⌘(⌃ 2 n X 1 2 i=1 2 kyi 1 W W xi k 2 Pn <latexit sha1_base64="xjOxvmjR64ptkxcnbTP+RZUHXVA=">AAACfXicfZDbahsxEIblTQ+Je3LSy96ImkApweyGkPQmEEgu2ou2KdRJIHKWWXnsiOiwlWaDzbLP0afpbfsMfZpUdlxondJBgo9/fjGavyi1CpSmP1vJyr37Dx6urrUfPX7y9FlnfeMkuMpL7EunnT8rIKBWFvukSONZ6RFMofG0uDqc9U+v0Qfl7GealjgwMLZqpCRQlPJOJggnFGiqkYtQmbxW+1lzYSe5iudCkCu5wC+Vuubv8vpDnjV5p5v20nnxu5AtoMsWdZyvt3bE0MnKoCWpIYTzLC1pUIMnJTU2bVEFLEFewRjPI1owGAb1fLeGb0ZlyEfOx2uJz9U/X9RgQpiaIjoN0GVY7s3Ef/YKszSZRm8GtbJlRWjl7eBRpTk5PguOD5VHSXoaAaRX8e9cXoIHSTHedlscYVzO4/s46GOJHsj517UAPzbKNnHZsdia0f+MMPltjBRzzpZTvQsn271st7f7aad7cLhIfJW9YC/ZK5axPXbA3rJj1meSfWXf2Hf2o3WTbCZbSe/WmrQWb56zvyrZ+wXtY8T8</latexit> > x x i=1 i i ⌘ IN1 x 1 <latexit sha1_base64="Fy4yLhPQfi77rKUpechpsDSdSpc=">AAACeXicfZBdSxtBFIYnaz9s+hX1sjfThkIqNexasb0pCHrRG9HSxghuEs5OTtbB+VhmzpaGJb/CX9Pb+iv8Ld50NqbQxtIDMzy85x3OnDcrlPQUx9eNaOXe/QcPVx81Hz95+ux5a239xNvSCewJq6w7zcCjkgZ7JEnhaeEQdKawn13s1/3+N3ReWvOVpgUONORGTqQACtKotZUeoCLg/eE2/8hTDNhJv8hcA9+qxf4weVOFazZMyRajVjvuxvPidyFZQJst6ni01thJx1aUGg0JBd6fJXFBgwocSaFw1kxLjwWIC8jxLKABjX5Qzfea8ddBGfOJdeEY4nP1zxcVaO+nOgtODXTul3u1+M9eppcm0+TDoJKmKAmNuB08KRUny+vQ+Fg6FKSmAUA4Gf7OxTk4EBSibTZDhmE5h4dh0FGBDsi6zSoFl2tpZmHZPH1b0/+M8P23MVDIOVlO9S6cbHeT3e7u55323v4i8VX2gr1iHZaw92yPfWLHrMcEu2Q/2E921biJXkadaPPWGjUWbzbYXxW9+wXRS8Ap</latexit> 1 W W )W (η : ) y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 1> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> 1 Wnew 2 Wnew <latexit sha1_base64="FnWtJpnmESiWqPWmMXLIvj3Hl/g=">AAACcXicfZDfShtBFMYn6//U2theFW8GgyBawm4R7aWgF70pKhgjuGk4OzlZB2dml5mzaljyBD6Nt+2T+By+gLMxgsbSAwO/+c43nDlfkivpKAwfasHM7Nz8wuJS/cPyx5VPjdXPZy4rrMC2yFRmzxNwqKTBNklSeJ5bBJ0o7CRXB1W/c43Wycyc0jDHrobUyIEUQF7qNTY6v6NeTHhLpcGbEY9TJMe9yLd5fIiKoLr0Gs2wFY6Lv4doAk02qePeam0n7mei0GhIKHDuIgpz6pZgSQqFo3pcOMxBXEGKFx4NaHTdcrzPiG94pc8HmfXHEB+rr1+UoJ0b6sQ7NdClm+5V4j97iZ6aTIMf3VKavCA04nnwoFCcMl6FxfvSoiA19ADCSv93Li7BgiAfab3uA/LLWfzlBx3laIEyu1XGYFMtzcgvm8bfKvqfEW5fjJ58ztF0qu/h7Hsr2m3tnuw09w8miS+yNbbONlnE9tg++8mOWZsJdsfu2R/2t/YYfA14sP5sDWqTN1/Ymwq2nwAwAr4q</latexit> <latexit sha1_base64="32vosqh6Ebm6Jfs/Vp/DHnn+K6g=">AAACcXicfZDRShtBFIYnW7WaVo16VXozGISiEnZFrJeCXnhTqmCM4Kbh7ORkHZyZXWbOtoYlT9Cn8VafpM/RF3A2RtAoHhj45j//cOb8Sa6kozD8Vws+zMzOfZxfqH/6vLi03FhZPXdZYQW2RaYye5GAQyUNtkmSwovcIuhEYSe5Pqz6nd9onczMGQ1z7GpIjRxIAeSlXmOj82unFxPeUGnwz4jHKZLjXuRbPD5CRVBdeo1m2ArHxV9DNIEmm9RJb6W2G/czUWg0JBQ4dxmFOXVLsCSFwlE9LhzmIK4hxUuPBjS6bjneZ8Q3vNLng8z6Y4iP1ecvStDODXXinRroyk33KvHNXqKnJtNgv1tKkxeERjwOHhSKU8arsHhfWhSkhh5AWOn/zsUVWBDkI63XfUB+OYs//KCfOVqgzG6WMdhUSzPyy6bxdkXvGeHmyejJ5xxNp/oaznda0V5r73S3eXA4SXyefWXr7BuL2Hd2wI7ZCWszwf6yW3bH7mv/gy8BD9YfrUFt8maNvahg6wE15r4t</latexit> 1 W + 2 W + W 1 W 2 小 行 力 入 二 4 1 行 行 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 力 ] 1 <latexit sha1_base64="z0c8cJk/WbV+FfELQ4+YHKO02PI=">AAACknicfVBdaxNBFJ2s1db40VR968tgEERK2E1LFZ8qFemDrVVMU+gmy93JTTp0ZnaZuSuGZf+Rv8Y30R/jbLoVTYsXLhzOPZdz70lzJR2F4Y9WcGvl9p3Vtbvte/cfPFzvbDw6cVlhBQ5EpjJ7moJDJQ0OSJLC09wi6FThML3Yr+fDL2idzMxnmuc40jAzcioFkKeSzrvhOOKxNDzWQOdpWn6qxuVR0o9JanT8KImqLT4c92/QbP/R9Kuk0w174aL4dRA1oMuaOk42WjvxJBOFRkNCgXNnUZjTqARLUiis2nHhMAdxATM889CAtxqVi4cr/swzEz7NrG9DfMH+vVGCdm6uU6+sT3bLs5q8cZbqJWeavhqV0uQFoRGXxtNCccp4nSafSIuC1NwDEFb627k4BwuCfObtdvwW/XMWD73RhxwtUGZflDHYmZam8s/O4q0a/U8IX6+EHvmco+VUr4OTfi/a7e1+3Onu7TeJr7FN9pQ9ZxF7yfbYATtmAybYN/ad/WS/gifB6+BN0GiDVrPzmP1TwfvfBADKbA==</latexit> 4 入 2 📄 2 N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
  19. NN む 2 <latexit sha1_base64="4QfpGZq2Y1weFxAfMEeD1Z2f4gg=">AAACVXicfZDbSgMxEIaz67me9dKbYBFEpOyKqDeCoBfeiArWCt0qs+m0huawJFmxLH0Mb/WxxIcRTGsFreJA4OOfP8zMn2aCWxdFb0E4Nj4xOTU9U5qdm19YXFpeubY6NwyrTAttblKwKLjCquNO4E1mEGQqsJZ2jvv92gMay7W6ct0MGxLairc4A+elepce0trtTu02frxbKkeVaFD0N8RDKJNhXdwtB7tJU7NconJMgLX1OMpcowDjOBPYKyW5xQxYB9pY96hAom0Ug517dMMrTdrSxj/l6ED9/qMAaW1Xpt4pwd3b0V5f/LOXypHJrnXQKLjKcoeKfQ5u5YI6TfuB0CY3yJzoegBmuN+dsnswwJyPrVRKTtAfZ/DMDzrP0IDTZqtIwLQlVz1/bDvZ7tN/Rnj8MnryOcejqf6G651KvFfZu9wtHx0PE58ma2SdbJKY7JMjckouSJUwoskTeSYvwWvwHo6Hk5/WMBj+WSU/Klz8AFWWtJs=</latexit> [Saxe-McClelland-Ganguli y=W W x N2

    ⇥N1 1 W 2R 2 ,W 2 R N3 ⇥N2 <latexit sha1_base64="I2c8rPXgJiVsk1hZFoxoKID7eqQ=">AAACe3icfZDfShtBFMYna1tt+i/qZW8GQ8GKTXeD/24EQQveSC00ruAky9nJ2Tg4M7vMzIphzWP4NN7qQ/gwBScxhTaWHhj48Z3vcOZ8aSGFdWH4UAvmXrx8Nb/wuv7m7bv3HxqLSyc2Lw3HDs9lbk5TsCiFxo4TTuJpYRBUKjFOL/bH/fgSjRW5/umGBXYVDLTIBAfnpaTx9dtq/JnuUpYZ4FGb2VIlldiNRj3NroeJoF9o3GvHvegqEey6104azbAVToo+h2gKTTKt42SxtsH6OS8VasclWHsWhYXrVmCc4BJHdVZaLIBfwADPPGpQaLvV5LIR/eSVPs1y4592dKL+OVGBsnaoUu9U4M7tbG8s/rOXqpnNLtvpVkIXpUPNnxZnpaQup+PYaF8Y5E4OPQA3wv+d8nPwmTkfbr3ODtAfZ/DIL/peoAGXm7WKgRkooUf+2AFbH9P/jHD12+jJ5xzNpvocTtqtaKu19WOjubc/TXyBfCQrZJVEZJvskUNyTDqEkxtyS+7Ife1X0AzWgvUna1CbziyTvyrYfAR1KsFl</latexit> n {(xi , yi )}i=1 <latexit sha1_base64="A0WnP2aUlxWqCCxSnZ7ftO30sLE=">AAACYXicfZBNS8NAEIa38avWr7biyUuwCCpSEhH1Igh68CIq2CqYGibbaV3c3YTdjbSE/Biv+os8+0fcfghaxYGFh3feZWbeKOFMG897LzhT0zOzc8X50sLi0vJKuVJt6jhVFBs05rG6i0AjZxIbhhmOd4lCEBHH2+jpdNC/fUalWSxvTD/BloCuZB1GwVgpLK8F2VYvZLv9kG0HeZixYz9/sHrNq3vDcn+DP4YaGddVWCnsB+2YpgKloRy0vve9xLQyUIZRjnkpSDUmQJ+gi/cWJQjUrWy4f+5uWqXtdmJlnzTuUP3+IwOhdV9E1inAPOrJ3kD8sxeJicmmc9TKmExSg5KOBndS7prYHYTjtplCanjfAlDF7O4ufQQF1NgIS6XgDO1xCi/soMsEFZhY7WQBqK5gMrfHdoPdAf1nhN6X0ZLN2Z9M9Tc09+r+Qf3ger92cjpOvEjWyQbZIj45JCfknFyRBqEkIy/klbwVPpx5p+xUR1anMP6zSn6Us/4J0564ng==</latexit> む E(W ) = <latexit sha1_base64="rd5oajX48R65mKNFLX36m+8U7g4=">AAACl3icfVDbahRBEO0db3G9bcyT+NK4CCKyzGiIIgQDCRIQY6JuEkjvDj29tZMmfRm6a8IOw3xUvkZ80z+xZ7OCbsSCgsOpU5yqkxVKeozj753o2vUbN2+t3O7euXvv/oPe6sNDb0snYCissu444x6UNDBEiQqOCwdcZwqOsrPtdn50Ds5La75iVcBI89zIqRQcA5X2PrAvMtecvt2kDGGGHisFlPlSp7XcTJqxqVI5S+WYoS0ok4YyzfE0y+rPzbjeS18xlBo83UuTJu3140E8L3oVJAvQJ4vaT1c762xiRanBoFDc+5MkLnBUc4dSKGi6rPRQcHHGczgJ0PBgNarnXzf0aWAmdGpdaIN0zv65UXPtfaWzoGxP9suzlvznLNNLzjh9M6qlKUoEIy6Np6WiaGkbKZ1IBwJVFQAXTobbqTjljgsMwXe7bAfCcw4+BqNPBTiO1j2vGXe5lqYJz+bsRYv+J+Sz38KAQs7JcqpXweHLQbIx2DhY729tLxJfIY/JE/KMJOQ12SK7ZJ8MiSAX5Bv5QX5Gj6J30fto91IadRY7a+Svig5+AeYTzfs=</latexit> ⌃ := む Pn > N3 ⇥N1 y x 2 R ; i=1 i i d 1 2> 2 1 ⌧ W = W (⌃ W W ) dt d 2 2 1 1> ⌧ W = (⌃ W W )W dt n X 1 2 i=1 2 kyi 1 W W xi k 2 Pn <latexit sha1_base64="xjOxvmjR64ptkxcnbTP+RZUHXVA=">AAACfXicfZDbahsxEIblTQ+Je3LSy96ImkApweyGkPQmEEgu2ou2KdRJIHKWWXnsiOiwlWaDzbLP0afpbfsMfZpUdlxondJBgo9/fjGavyi1CpSmP1vJyr37Dx6urrUfPX7y9FlnfeMkuMpL7EunnT8rIKBWFvukSONZ6RFMofG0uDqc9U+v0Qfl7GealjgwMLZqpCRQlPJOJggnFGiqkYtQmbxW+1lzYSe5iudCkCu5wC+Vuubv8vpDnjV5p5v20nnxu5AtoMsWdZyvt3bE0MnKoCWpIYTzLC1pUIMnJTU2bVEFLEFewRjPI1owGAb1fLeGb0ZlyEfOx2uJz9U/X9RgQpiaIjoN0GVY7s3Ef/YKszSZRm8GtbJlRWjl7eBRpTk5PguOD5VHSXoaAaRX8e9cXoIHSTHedlscYVzO4/s46GOJHsj517UAPzbKNnHZsdia0f+MMPltjBRzzpZTvQsn271st7f7aad7cLhIfJW9YC/ZK5axPXbA3rJj1meSfWXf2Hf2o3WTbCZbSe/WmrQWb56zvyrZ+wXtY8T8</latexit> > x x i=1 i i ⌘ IN1 y <latexit sha1_base64="aoewlSBH5BQHYAHU0t//VBdVAGs=">AAACf3icfVBdaxNBFJ2sX238Su1jXwaDWEXDbixVHwqF+uCLWNE0hW4S7s7e3Q6dj2XmrhiW/SH+Gl/1J/hvnKQRNBUvDBzOOZcz92SVkp7i+Gcnunb9xs1bG5vd23fu3rvf23pw4m3tBI6EVdadZuBRSYMjkqTwtHIIOlM4zi6OFvr4MzovrflE8wonGkojCymAAjXrvUgJap4WDkSTt01OLR9PE37Am/F02E5TstVu+lGWGvjzoAyD+GTW68eDeDn8KkhWoM9Wczzb6uyluRW1RkNCgfdnSVzRpAFHUihsu2ntsQJxASWeBWhAo580y+ta/igwOS+sC88QX7J/bjSgvZ/rLDg10Llf1xbkP7VMryVT8WrSSFPVhEZcBhe14mT5ojqeS4eC1DwAEE6Gv3NxDqE5CgV3u+kbDMc5fBeC3lfogKx72qTgSi1NG44t02cL9D8jfPltDCj0nKy3ehWcDAfJ/mD/w17/8GjV+AbbYQ/ZLkvYS3bI3rJjNmKCfWXf2Hf2I+pEj6NBFF9ao85qZ5v9NdHrX7oqwlQ=</latexit> <latexit sha1_base64="eTyooeVi8+L+NaNjtnmt/dC42d8=">AAACf3icfVDbbtNAEN24XEq4peWRlxURoiCI7FIVeECqVB54QRRBmkp1Eo3XY3fVvVi7Y0Rk+UP4Gl7hE/gb1mmQIEWMtKuzZ87o7JysUtJTHP/sRRtXrl67vnmjf/PW7Tt3B1vbx97WTuBYWGXdSQYelTQ4JkkKTyqHoDOFk+z8sOtPPqPz0ppPtKhwqqE0spACKFDzwfOUoOZp4UA0edvk1PLJbJe/5jvpR1lq4M+692SWPG7C1c5SstV8MIxH8bL4ZZCswJCt6mi+1dtLcytqjYaEAu9Pk7iiaQOOpFDY9tPaYwXiHEo8DdCARj9tltu1/GFgcl5YF44hvmT/nGhAe7/QWVBqoDO/3uvIf/YyveZMxctpI01VExpxYVzUipPlXXQ8lw4FqUUAIJwMf+fiDEJyFALu99M3GJZz+C4Yva/QAVn3pEnBlVqaNixbpk879D8hfPktDCjknKynehkc746S/dH+h73hweEq8U12nz1gOyxhL9gBe8uO2JgJ9pV9Y9/Zj6gXPYpGUXwhjXqrmXvsr4pe/QKoBsJU</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> (τ = η −1) <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> 1 Wnew 2 Wnew <latexit sha1_base64="FnWtJpnmESiWqPWmMXLIvj3Hl/g=">AAACcXicfZDfShtBFMYn6//U2theFW8GgyBawm4R7aWgF70pKhgjuGk4OzlZB2dml5mzaljyBD6Nt+2T+By+gLMxgsbSAwO/+c43nDlfkivpKAwfasHM7Nz8wuJS/cPyx5VPjdXPZy4rrMC2yFRmzxNwqKTBNklSeJ5bBJ0o7CRXB1W/c43Wycyc0jDHrobUyIEUQF7qNTY6v6NeTHhLpcGbEY9TJMe9yLd5fIiKoLr0Gs2wFY6Lv4doAk02qePeam0n7mei0GhIKHDuIgpz6pZgSQqFo3pcOMxBXEGKFx4NaHTdcrzPiG94pc8HmfXHEB+rr1+UoJ0b6sQ7NdClm+5V4j97iZ6aTIMf3VKavCA04nnwoFCcMl6FxfvSoiA19ADCSv93Li7BgiAfab3uA/LLWfzlBx3laIEyu1XGYFMtzcgvm8bfKvqfEW5fjJ58ztF0qu/h7Hsr2m3tnuw09w8miS+yNbbONlnE9tg++8mOWZsJdsfu2R/2t/YYfA14sP5sDWqTN1/Ymwq2nwAwAr4q</latexit> <latexit sha1_base64="32vosqh6Ebm6Jfs/Vp/DHnn+K6g=">AAACcXicfZDRShtBFIYnW7WaVo16VXozGISiEnZFrJeCXnhTqmCM4Kbh7ORkHZyZXWbOtoYlT9Cn8VafpM/RF3A2RtAoHhj45j//cOb8Sa6kozD8Vws+zMzOfZxfqH/6vLi03FhZPXdZYQW2RaYye5GAQyUNtkmSwovcIuhEYSe5Pqz6nd9onczMGQ1z7GpIjRxIAeSlXmOj82unFxPeUGnwz4jHKZLjXuRbPD5CRVBdeo1m2ArHxV9DNIEmm9RJb6W2G/czUWg0JBQ4dxmFOXVLsCSFwlE9LhzmIK4hxUuPBjS6bjneZ8Q3vNLng8z6Y4iP1ecvStDODXXinRroyk33KvHNXqKnJtNgv1tKkxeERjwOHhSKU8arsHhfWhSkhh5AWOn/zsUVWBDkI63XfUB+OYs//KCfOVqgzG6WMdhUSzPyy6bxdkXvGeHmyejJ5xxNp/oaznda0V5r73S3eXA4SXyefWXr7BuL2Hd2wI7ZCWszwf6yW3bH7mv/gy8BD9YfrUFt8maNvahg6wE15r4t</latexit> 1 W + 2 W + W 1 W 2 小 行 力 入 二 4 1 行 行 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 力 ] 1 <latexit sha1_base64="z0c8cJk/WbV+FfELQ4+YHKO02PI=">AAACknicfVBdaxNBFJ2s1db40VR968tgEERK2E1LFZ8qFemDrVVMU+gmy93JTTp0ZnaZuSuGZf+Rv8Y30R/jbLoVTYsXLhzOPZdz70lzJR2F4Y9WcGvl9p3Vtbvte/cfPFzvbDw6cVlhBQ5EpjJ7moJDJQ0OSJLC09wi6FThML3Yr+fDL2idzMxnmuc40jAzcioFkKeSzrvhOOKxNDzWQOdpWn6qxuVR0o9JanT8KImqLT4c92/QbP/R9Kuk0w174aL4dRA1oMuaOk42WjvxJBOFRkNCgXNnUZjTqARLUiis2nHhMAdxATM889CAtxqVi4cr/swzEz7NrG9DfMH+vVGCdm6uU6+sT3bLs5q8cZbqJWeavhqV0uQFoRGXxtNCccp4nSafSIuC1NwDEFb627k4BwuCfObtdvwW/XMWD73RhxwtUGZflDHYmZam8s/O4q0a/U8IX6+EHvmco+VUr4OTfi/a7e1+3Onu7TeJr7FN9pQ9ZxF7yfbYATtmAybYN/ad/WS/gifB6+BN0GiDVrPzmP1TwfvfBADKbA==</latexit> 5 入 2 📄 2 N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
  20. ] = む む <latexit sha1_base64="UkkM6NJBPc2z1p1wXPHy0oP+PGU=">AAAClHicfZBdaxNBFIYn61eNX2kL3ngzGAQRCbulVC9aKKRCb6yVmrTQTZezk5PN0PliZrYYlv1L/TW98UJ/i5NkC5qKBwYe3vMezpw3N4I7H8c/WtG9+w8ePlp73H7y9NnzF531jaHTpWU4YFpoe5aDQ8EVDjz3As+MRZC5wNP8sj/vn16hdVyrb35mcCShUHzCGfggZZ3D9IQXEugeHZwML1KvTcDUlTKrUhBmCntJfVEdZUntsqVAy1u4amAxlnW6cS9eFL0LSQNd0tRxtt7aTsealRKVZwKcO09i40cVWM+ZwLqdlg4NsEso8DygAoluVC1OrumboIzpRNvwlKcL9c+JCqRzM5kHpwQ/dau9ufjPXi5XNvvJx1HFlSk9KrZcPCkF9ZrO86RjbpF5MQsAzPLwd8qmYIH5kHq7nR5gOM7i57Doi0ELXtt3IVpbSK7qcGyRvp/T/4zw/dYYKOScrKZ6F4ZbvWSnt/N1u7vfbxJfI6/Ia/KWJOQD2SeH5JgMCCPX5Ib8JL+il9Fu1I8+La1Rq5nZJH9VdPQb5EDMQQ==</latexit> ⌃ = U SV

    > = N1 X > s ↵ u↵ v ↵ ; ↵=1 N 1  N3 <latexit sha1_base64="tdzJLIFIDwXliskwx2dHNvWnIis=">AAACVnicfZDbSgMxEIaz67meql56EyyCiJRdFfVS0Atv1Aq2FdxSZtNpDU2yS5IVy9LX8FYfS19GTA+CVnEg8PHPH2bmj1PBjQ2Cd8+fmp6ZnZtfKCwuLa+sFtfWaybJNMMqS0Si72IwKLjCquVW4F2qEWQssB53zwb9+iNqwxN1a3spNiR0FG9zBtZJ0VUzpJFAetU8oM1iKSgHw6K/IRxDiYyr0lzzDqNWwjKJyjIBxtyHQWobOWjLmcB+IcoMpsC60MF7hwokmkY+XLpPt53Sou1Eu6csHarff+QgjenJ2Dkl2Acz2RuIf/ZiOTHZtk8aOVdpZlGx0eB2JqhN6CAR2uIamRU9B8A0d7tT9gAamHW5FQrRObrjNF66QdcparCJ3s0j0B3JVd8d24n2BvSfEZ6+jI5czuFkqr+htl8Oj8pHN4el07Nx4vNkk2yRHRKSY3JKLkiFVAkjKXkmL+TVe/M+/Bl/bmT1vfGfDfKj/OInnU60tQ==</latexit> (wlog) む d 1 2> ⌧ W = W (⌃ dt <latexit sha1_base64="aoewlSBH5BQHYAHU0t//VBdVAGs=">AAACf3icfVBdaxNBFJ2sX238Su1jXwaDWEXDbixVHwqF+uCLWNE0hW4S7s7e3Q6dj2XmrhiW/SH+Gl/1J/hvnKQRNBUvDBzOOZcz92SVkp7i+Gcnunb9xs1bG5vd23fu3rvf23pw4m3tBI6EVdadZuBRSYMjkqTwtHIIOlM4zi6OFvr4MzovrflE8wonGkojCymAAjXrvUgJap4WDkSTt01OLR9PE37Am/F02E5TstVu+lGWGvjzoAyD+GTW68eDeDn8KkhWoM9Wczzb6uyluRW1RkNCgfdnSVzRpAFHUihsu2ntsQJxASWeBWhAo580y+ta/igwOS+sC88QX7J/bjSgvZ/rLDg10Llf1xbkP7VMryVT8WrSSFPVhEZcBhe14mT5ojqeS4eC1DwAEE6Gv3NxDqE5CgV3u+kbDMc5fBeC3lfogKx72qTgSi1NG44t02cL9D8jfPltDCj0nKy3ehWcDAfJ/mD/w17/8GjV+AbbYQ/ZLkvYS3bI3rJjNmKCfWXf2Hf2I+pEj6NBFF9ao85qZ5v9NdHrX7oqwlQ=</latexit> d 2 ⌧ W = (⌃ dt <latexit sha1_base64="eTyooeVi8+L+NaNjtnmt/dC42d8=">AAACf3icfVDbbtNAEN24XEq4peWRlxURoiCI7FIVeECqVB54QRRBmkp1Eo3XY3fVvVi7Y0Rk+UP4Gl7hE/gb1mmQIEWMtKuzZ87o7JysUtJTHP/sRRtXrl67vnmjf/PW7Tt3B1vbx97WTuBYWGXdSQYelTQ4JkkKTyqHoDOFk+z8sOtPPqPz0ppPtKhwqqE0spACKFDzwfOUoOZp4UA0edvk1PLJbJe/5jvpR1lq4M+692SWPG7C1c5SstV8MIxH8bL4ZZCswJCt6mi+1dtLcytqjYaEAu9Pk7iiaQOOpFDY9tPaYwXiHEo8DdCARj9tltu1/GFgcl5YF44hvmT/nGhAe7/QWVBqoDO/3uvIf/YyveZMxctpI01VExpxYVzUipPlXXQ8lw4FqUUAIJwMf+fiDEJyFALu99M3GJZz+C4Yva/QAVn3pEnBlVqaNixbpk879D8hfPktDCjknKynehkc746S/dH+h73hweEq8U12nz1gOyxhL9gBe8uO2JgJ9pV9Y9/Zj6gXPYpGUXwhjXqrmXvsr4pe/QKoBsJU</latexit> 2 1 W W ) 1 <latexit sha1_base64="lADZqvD1jq4EQoZ3DU7noDJGtks=">AAACZnicfVBdSxtBFJ2stbWp1WgphfZlaCiUUsJuEfVFEPTBF6mFJhHcGO5ObuLgfCwzd8WwBPw1fdW/4z/wZzgbU2ijeGHgzLnnzpl7slxJT3F8W4sWXiy+fLX0uv5m+e3KamNtveNt4QS2hVXWHWfgUUmDbZKk8Dh3CDpT2M3O96p+9wKdl9b8pnGOPQ0jI4dSAAWq3/jYPU34Dk9tEFVv8OreOU3J5v1GM27F0+KPQTIDTTaro/5abSMdWFFoNCQUeH+SxDn1SnAkhcJJPS085iDOYYQnARrQ6HvldIkJ/xKYAR9aF44hPmX/nShBez/WWVBqoDM/36vIJ3uZnnOm4XavlCYvCI14MB4WipPlVUJ8IB0KUuMAQDgZ/s7FGTgQFHKs19N9DMs5PAxGP3N0QNZ9K1NwIy3NJCw7Sr9X6DkhXP4VBhRyTuZTfQw6P1rJZmvz10Zzd2+W+BL7xD6zryxhW2yXHbAj1maCXbE/7Jrd1O6ileh99OFBGtVmM+/YfxXxe5l4udQ=</latexit> 2 d 1 2> ⌧ W = W (S dt W2 = UW <latexit sha1_base64="vtqRQqNagU2/CqmTU+eODBmQnrE=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJae3nvj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X6+Cs6r6z5QosSpwXMjcqVBArUrDcSBJXIHcg6a+qMGmGDvb2Nj08T/prXl4m95lSQLfnuZ/6CrwlrbU9nvX48iJfFr4JkBfpsVcPZTmdfZFZWBRqSGryfJHFJ0xocKamx6YrKYwnyHOY4CdBAgX5aL/dv+JPAZDy3LhxDfMle7qih8H5RpMFZAJ35Ta0l/6mlxcZkyl9Na2XKitDIi8F5pTlZ3obLM+VQkl4EANKp8HYuzyCES+ELul3xBsNyDt+HQR9KdEDWPasFuHmhTBOWnYvnLfqfEb79NQYUck42U70KjvcGycHg4ON+//BolfgWe8Qes12WsJfskL1lQzZikn1nP9kv9jvqR++iYfTpwhp1Vj0P2VpFkz+0sNEO</latexit> d 2 ⌧ W = (S dt 1 W W )W <latexit sha1_base64="nxrvmp5jtcwtpIwUs1pWzkUlXgE=">AAACYHicfVBdSxtBFJ2stY3R1kR9al8Gg1BKCbsi1hdB0AdfSlMwiZCN4e7kJg7OxzIzK8Yl/8VX/Ue++kucTVKoUbwwcDj33Dn3niQV3LowfCwFSx+WP34qr1RW1z5/Wa/WNtpWZ4Zhi2mhzXkCFgVX2HLcCTxPDYJMBHaSq+Oi37lGY7lWZ26cYk/CSPEhZ+A81a9udS526SFtxdqrik+oJ/rVetgIp0Vfg2gO6mRezX6ttBcPNMskKscEWNuNwtT1cjCOM4GTSpxZTIFdwQi7HiqQaHv5dP0J3fHMgA618U85OmX/n8hBWjuWiVdKcJd2sVeQb/YSueDshge9nKs0c6jYzHiYCeo0LbKhA26QOTH2AJjhfnfKLsEAcz7BSiU+QX+cwd/e6E+KBpw2P/IYzEhyNfHHjuKfBXpPCDf/hB75nKPFVF+D9m4j2m/s/92rHx3PEy+Tb2SbfCcR+UWOyClpkhZh5JbckXvyUHoKysF6UJtJg9J8ZpO8qODrM2pot2w=</latexit> W1 = W V > 1> x N2 <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 2 y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 <latexit sha1_base64="JvAcOgrCKAA5nIOCF0i0oH2858o=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJbevHnj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X18Vd0XlnzhRYlTguYG5UrCRSoWW8kCCqRO5B11tQZNcIGeXsbH5/u8dd89zN/wdfIy0nytF5Lm1NBtuSzXj8exMvgV0GyAn22iuFsp7MvMiurAg1JDd5PkrikaQ2OlNTYdEXlsQR5DnOcBGigQD+tl/s3/ElgMp5bF44hvmQvd9RQeL8o0qAsgM78Zq0l/1lLi43JlL+a1sqUFaGRF4PzSnOyvDWXZ8qhJL0IAKRT4e1cnkEwl8IXdLviDYblHL4Pgz6U6ICse1YLcPNCmSYsOxfPW/Q/IXz7Kwwo+JxsunoVHO8NkoPBwcf9/uHRyvEt9og9ZrssYS/ZIXvLhmzEJPvOfrJf7HfUj95Fw+jThTTqrHoesrWIJn8AllfRDg==</latexit> 2 1 W W ) 2 1 W W )W 1> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> 力 入 行 4 1 行 行 力 入 力 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 6 入 2 📄 [Saxe-McClelland-Ganguli N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
  21. d 1 2> ⌧ W = W (⌃ dt d

    2 ⌧ W = (⌃ dt <latexit sha1_base64="aoewlSBH5BQHYAHU0t//VBdVAGs=">AAACf3icfVBdaxNBFJ2sX238Su1jXwaDWEXDbixVHwqF+uCLWNE0hW4S7s7e3Q6dj2XmrhiW/SH+Gl/1J/hvnKQRNBUvDBzOOZcz92SVkp7i+Gcnunb9xs1bG5vd23fu3rvf23pw4m3tBI6EVdadZuBRSYMjkqTwtHIIOlM4zi6OFvr4MzovrflE8wonGkojCymAAjXrvUgJap4WDkSTt01OLR9PE37Am/F02E5TstVu+lGWGvjzoAyD+GTW68eDeDn8KkhWoM9Wczzb6uyluRW1RkNCgfdnSVzRpAFHUihsu2ntsQJxASWeBWhAo580y+ta/igwOS+sC88QX7J/bjSgvZ/rLDg10Llf1xbkP7VMryVT8WrSSFPVhEZcBhe14mT5ojqeS4eC1DwAEE6Gv3NxDqE5CgV3u+kbDMc5fBeC3lfogKx72qTgSi1NG44t02cL9D8jfPltDCj0nKy3ehWcDAfJ/mD/w17/8GjV+AbbYQ/ZLkvYS3bI3rJjNmKCfWXf2Hf2I+pEj6NBFF9ao85qZ5v9NdHrX7oqwlQ=</latexit> 1 2 1 W W ) 2 > d 1 2> ⌧ W = W (S dt 2 W = UW d 2 ⌧ W = (S dt <latexit sha1_base64="vtqRQqNagU2/CqmTU+eODBmQnrE=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJae3nvj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X6+Cs6r6z5QosSpwXMjcqVBArUrDcSBJXIHcg6a+qMGmGDvb2Nj08T/prXl4m95lSQLfnuZ/6CrwlrbU9nvX48iJfFr4JkBfpsVcPZTmdfZFZWBRqSGryfJHFJ0xocKamx6YrKYwnyHOY4CdBAgX5aL/dv+JPAZDy3LhxDfMle7qih8H5RpMFZAJ35Ta0l/6mlxcZkyl9Na2XKitDIi8F5pTlZ3obLM+VQkl4EANKp8HYuzyCES+ELul3xBsNyDt+HQR9KdEDWPasFuHmhTBOWnYvnLfqfEb79NQYUck42U70KjvcGycHg4ON+//BolfgWe8Qes12WsJfskL1lQzZikn1nP9kv9jvqR++iYfTpwhp1Vj0P2VpFkz+0sNEO</latexit> 1 W W )W <latexit sha1_base64="nxrvmp5jtcwtpIwUs1pWzkUlXgE=">AAACYHicfVBdSxtBFJ2stY3R1kR9al8Gg1BKCbsi1hdB0AdfSlMwiZCN4e7kJg7OxzIzK8Yl/8VX/Ue++kucTVKoUbwwcDj33Dn3niQV3LowfCwFSx+WP34qr1RW1z5/Wa/WNtpWZ4Zhi2mhzXkCFgVX2HLcCTxPDYJMBHaSq+Oi37lGY7lWZ26cYk/CSPEhZ+A81a9udS526SFtxdqrik+oJ/rVetgIp0Vfg2gO6mRezX6ttBcPNMskKscEWNuNwtT1cjCOM4GTSpxZTIFdwQi7HiqQaHv5dP0J3fHMgA618U85OmX/n8hBWjuWiVdKcJd2sVeQb/YSueDshge9nKs0c6jYzHiYCeo0LbKhA26QOTH2AJjhfnfKLsEAcz7BSiU+QX+cwd/e6E+KBpw2P/IYzEhyNfHHjuKfBXpPCDf/hB75nKPFVF+D9m4j2m/s/92rHx3PEy+Tb2SbfCcR+UWOyClpkhZh5JbckXvyUHoKysF6UJtJg9J8ZpO8qODrM2pot2w=</latexit> W =W V 1> 2 <latexit sha1_base64="JvAcOgrCKAA5nIOCF0i0oH2858o=">AAACoXicfVFNbxMxEHWWrxI+msKRi0WEVBBEu1VVuCBVKgc4IMJHmkpxGs16Z1OrXntlzyKi1f4yfglHrvAn8KZBalLESJbevHnj8TynpVae4vhHJ7p2/cbNW1u3u3fu3ru/3dt5cOxt5SSOpNXWnaTgUSuDI1Kk8aR0CEWqcZyeH7X18Vd0XlnzhRYlTguYG5UrCRSoWW8kCCqRO5B11tQZNcIGeXsbH5/u8dd89zN/wdfIy0nytF5Lm1NBtuSzXj8exMvgV0GyAn22iuFsp7MvMiurAg1JDd5PkrikaQ2OlNTYdEXlsQR5DnOcBGigQD+tl/s3/ElgMp5bF44hvmQvd9RQeL8o0qAsgM78Zq0l/1lLi43JlL+a1sqUFaGRF4PzSnOyvDWXZ8qhJL0IAKRT4e1cnkEwl8IXdLviDYblHL4Pgz6U6ICse1YLcPNCmSYsOxfPW/Q/IXz7Kwwo+JxsunoVHO8NkoPBwcf9/uHRyvEt9og9ZrssYS/ZIXvLhmzEJPvOfrJf7HfUj95Fw+jThTTqrHoesrWIJn8AllfRDg==</latexit> 2 1 W W ) 2 1 W W )W ↵ 1> ↵ N2 b 2R <latexit sha1_base64="DHLyS4uYdl0Lg3kV3tIdx5ykrr4=">AAACaXicfZDfSiMxFMbT8c+6dXXb9UYUIVgWFpEyI+LupeBeeONfrApOLSfpaRtMMkOSWSzDXPk0e6tPs8/gS5ipFdYqHgj8+M4XzjkfS6WwLgz/VYKp6ZnZT3Ofq/NfFha/1urfzm2SGY4tnsjEXDKwKIXGlhNO4mVqEBSTeMFu9sr+xR80ViT6zA1TbCvoa9ETHJyXOrU1dh2DTAdAY6FprMANGMtPi+v8sLNVdGqNsBmOir6FaAwNMq7jTr2yHXcTninUjkuw9ioKU9fOwTjBJRbVOLOYAr+BPl551KDQtvPRHQX97pUu7SXGP+3oSP3/Rw7K2qFi3lkuaid7pfhuj6mJya73q50LnWYONX8e3MskdQktQ6JdYZA7OfQA3Ai/O+UDMMCdj7JajX+jP87ggR90lKIBl5iNPAbTV0IX/th+vFnSR0a4fTF68jlHk6m+hfOtZrTT3DnZbuzujROfI6tknfwgEflJdsk+OSYtwskd+UvuyUPlMagHy8HKszWojP8skVcVNJ4AZ0C8JA==</latexit> N2 a 2R <latexit sha1_base64="DdEbmgm8rji84Qf/R9oPSY2wd5c=">AAACaXicfZDbSiNBEIY742HdeNhEb0QRGoMgImFGRL0U3AtvPGJUcGKo6VSSxu6eobtHDMNc+TR7q0+zz+BL2BMjaBQLGj7++puq+qNEcGN9/3/JGxufmPw19bs8PTM796dSnb80caoZNlgsYn0dgUHBFTYstwKvE40gI4FX0d1B0b+6R214rC5sP8GmhK7iHc7AOqlVWYHbEETSAxpyRUMJthdF2Xl+mx23tvJWpebX/UHRrxAMoUaGddqqlrbDdsxSicoyAcbcBH5imxloy5nAvBymBhNgd9DFG4cKJJpmNrgjp2tOadNOrN1Tlg7Ujz8ykMb0ZeScxaJmtFeI3/YiOTLZdvaaGVdJalGxt8GdVFAb0yIk2uYamRV9B8A0d7tT1gMNzLooy+XwL7rjNB65QScJarCx3shC0F3JVe6O7YabBf1khId3oyOXczCa6le43KoHO/Wds+3a/sEw8SmyTFbJOgnILtknh+SUNAgjj+QfeSLPpRev6i16S29WrzT8s0A+lVd7BWU9vCM=</latexit> x 1 <latexit sha1_base64="2N2HhVHrFmNqTj/W7r8zUXVj4gk=">AAACWHicfVBdSxtREJ2sbaOxtkYffbk0CKVI2BWxvgiCffCl1IIxghtl9mYSL7kfy713xbDkf/ha/5X+Gu/GCDWKAwOHc84wMyfLpXA+ju9r0cKHj5/qi0uN5c8rX76uNtdOnSkspw430tizDB1JoanjhZd0lltClUnqZqPDSu9ek3XC6BM/zqmncKjFQHD0gbpITRCrWda92N6/XG3F7Xha7DVIZqAFszq+bNZ20r7hhSLtuUTnzpM4970SrRdc0qSRFo5y5CMc0nmAGhW5Xjk9e8I2A9NnA2NDa8+m7P8TJSrnxioLToX+ys1rFfmmlqm5zX6w1yuFzgtPmj8tHhSSecOqTFhfWOJejgNAbkW4nfErtMh9SK7RSH9ReM7S77DoT04WvbE/yhTtUAk9Cc8O060KvWfEm2djQCHnZD7V1+B0u53stnf/7rQODmeJL8IGfIPvkMBPOIAjOIYOcLBwC//grvYQQVSPlp6sUW02sw4vKlp7BBtXtYM=</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> W = 2 N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit> W = y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="eC1WdBSeIwB8HNfOlRvzX95FnW0=">AAACWHicfVDbSiNBEK3Mrrd410dfmg2CiIQZEd2XBUEffBEVjBGcKDWdSmzSl6G7Z9kw5D/21f2r3a/ZnhhBo1hQcDjnFFV1slwK5+P4by368nVmdm5+ob64tLyyura+ceNMYTm1uJHG3mboSApNLS+8pNvcEqpMUjsbnFR6+ydZJ4y+9sOcOgr7WvQERx+o+9QEsZpl7fvkx8NaI27G42LvQTIBDZjU5cN67SDtGl4o0p5LdO4uiXPfKdF6wSWN6mnhKEc+wD7dBahRkeuU47NHbDswXdYzNrT2bMy+nihROTdUWXAq9I9uWqvID7VMTW32ve+dUui88KT58+JeIZk3rMqEdYUl7uUwAORWhNsZf0SL3Ifk6vX0lMJzls7DooucLHpjd8sUbV8JPQrP9tO9Cn1mxF8vxoBCzsl0qu/BzX4zOWweXh00jk8mic/DFnyDHUjgCI7hDC6hBRws/IYn+FP7F0E0Fy08W6PaZGYT3lS08R8Zb7WC</latexit> ] <latexit sha1_base64="eTyooeVi8+L+NaNjtnmt/dC42d8=">AAACf3icfVDbbtNAEN24XEq4peWRlxURoiCI7FIVeECqVB54QRRBmkp1Eo3XY3fVvVi7Y0Rk+UP4Gl7hE/gb1mmQIEWMtKuzZ87o7JysUtJTHP/sRRtXrl67vnmjf/PW7Tt3B1vbx97WTuBYWGXdSQYelTQ4JkkKTyqHoDOFk+z8sOtPPqPz0ppPtKhwqqE0spACKFDzwfOUoOZp4UA0edvk1PLJbJe/5jvpR1lq4M+692SWPG7C1c5SstV8MIxH8bL4ZZCswJCt6mi+1dtLcytqjYaEAu9Pk7iiaQOOpFDY9tPaYwXiHEo8DdCARj9tltu1/GFgcl5YF44hvmT/nGhAe7/QWVBqoDO/3uvIf/YyveZMxctpI01VExpxYVzUipPlXXQ8lw4FqUUAIJwMf+fiDEJyFALu99M3GJZz+C4Yva/QAVn3pEnBlVqaNixbpk879D8hfPktDCjknKynehkc746S/dH+h73hweEq8U12nz1gOyxhL9gBe8uO2JgJ9pV9Y9/Zj6gXPYpGUXwhjXqrmXvsr4pe/QKoBsJU</latexit> 1 <latexit sha1_base64="lADZqvD1jq4EQoZ3DU7noDJGtks=">AAACZnicfVBdSxtBFJ2stbWp1WgphfZlaCiUUsJuEfVFEPTBF6mFJhHcGO5ObuLgfCwzd8WwBPw1fdW/4z/wZzgbU2ijeGHgzLnnzpl7slxJT3F8W4sWXiy+fLX0uv5m+e3KamNtveNt4QS2hVXWHWfgUUmDbZKk8Dh3CDpT2M3O96p+9wKdl9b8pnGOPQ0jI4dSAAWq3/jYPU34Dk9tEFVv8OreOU3J5v1GM27F0+KPQTIDTTaro/5abSMdWFFoNCQUeH+SxDn1SnAkhcJJPS085iDOYYQnARrQ6HvldIkJ/xKYAR9aF44hPmX/nShBez/WWVBqoDM/36vIJ3uZnnOm4XavlCYvCI14MB4WipPlVUJ8IB0KUuMAQDgZ/s7FGTgQFHKs19N9DMs5PAxGP3N0QNZ9K1NwIy3NJCw7Sr9X6DkhXP4VBhRyTuZTfQw6P1rJZmvz10Zzd2+W+BL7xD6zryxhW2yXHbAj1maCXbE/7Jrd1O6ileh99OFBGtVmM+/YfxXxe5l4udQ=</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> d ↵ ⌧ a = (s↵ dt <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ↵ ↵ ha , b i)b ↵ X 6=↵ d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> ↵ b ha , b i ↵ ↵ ha , b i)a ↵ X 6=↵ ↵ b ha , b i 4 1 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 7 2 📄 [Saxe-McClelland-Ganguli N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
  22. 1 d ↵ ⌧ a = (s↵ dt X ha↵

    , b↵ i)b↵ ↵ 6=↵ N2 a 2R <latexit sha1_base64="DdEbmgm8rji84Qf/R9oPSY2wd5c=">AAACaXicfZDbSiNBEIY742HdeNhEb0QRGoMgImFGRL0U3AtvPGJUcGKo6VSSxu6eobtHDMNc+TR7q0+zz+BL2BMjaBQLGj7++puq+qNEcGN9/3/JGxufmPw19bs8PTM796dSnb80caoZNlgsYn0dgUHBFTYstwKvE40gI4FX0d1B0b+6R214rC5sP8GmhK7iHc7AOqlVWYHbEETSAxpyRUMJthdF2Xl+mx23tvJWpebX/UHRrxAMoUaGddqqlrbDdsxSicoyAcbcBH5imxloy5nAvBymBhNgd9DFG4cKJJpmNrgjp2tOadNOrN1Tlg7Ujz8ykMb0ZeScxaJmtFeI3/YiOTLZdvaaGVdJalGxt8GdVFAb0yIk2uYamRV9B8A0d7tT1gMNzLooy+XwL7rjNB65QScJarCx3shC0F3JVe6O7YabBf1khId3oyOXczCa6le43KoHO/Wds+3a/sEw8SmyTFbJOgnILtknh+SUNAgjj+QfeSLPpRev6i16S29WrzT8s0A+lVd7BWU9vCM=</latexit> <latexit sha1_base64="eC1WdBSeIwB8HNfOlRvzX95FnW0=">AAACWHicfVDbSiNBEK3Mrrd410dfmg2CiIQZEd2XBUEffBEVjBGcKDWdSmzSl6G7Z9kw5D/21f2r3a/ZnhhBo1hQcDjnFFV1slwK5+P4by368nVmdm5+ob64tLyyura+ceNMYTm1uJHG3mboSApNLS+8pNvcEqpMUjsbnFR6+ydZJ4y+9sOcOgr7WvQERx+o+9QEsZpl7fvkx8NaI27G42LvQTIBDZjU5cN67SDtGl4o0p5LdO4uiXPfKdF6wSWN6mnhKEc+wD7dBahRkeuU47NHbDswXdYzNrT2bMy+nihROTdUWXAq9I9uWqvID7VMTW32ve+dUui88KT58+JeIZk3rMqEdYUl7uUwAORWhNsZf0SL3Ifk6vX0lMJzls7DooucLHpjd8sUbV8JPQrP9tO9Cn1mxF8vxoBCzsl0qu/BzX4zOWweXh00jk8mic/DFnyDHUjgCI7hDC6hBRws/IYn+FP7F0E0Fy08W6PaZGYT3lS08R8Zb7WC</latexit> 1 d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> b ha↵ , b i ha↵ , b↵ i)a↵ ↵ N2 b 2R <latexit sha1_base64="DHLyS4uYdl0Lg3kV3tIdx5ykrr4=">AAACaXicfZDfSiMxFMbT8c+6dXXb9UYUIVgWFpEyI+LupeBeeONfrApOLSfpaRtMMkOSWSzDXPk0e6tPs8/gS5ipFdYqHgj8+M4XzjkfS6WwLgz/VYKp6ZnZT3Ofq/NfFha/1urfzm2SGY4tnsjEXDKwKIXGlhNO4mVqEBSTeMFu9sr+xR80ViT6zA1TbCvoa9ETHJyXOrU1dh2DTAdAY6FprMANGMtPi+v8sLNVdGqNsBmOir6FaAwNMq7jTr2yHXcTninUjkuw9ioKU9fOwTjBJRbVOLOYAr+BPl551KDQtvPRHQX97pUu7SXGP+3oSP3/Rw7K2qFi3lkuaid7pfhuj6mJya73q50LnWYONX8e3MskdQktQ6JdYZA7OfQA3Ai/O+UDMMCdj7JajX+jP87ggR90lKIBl5iNPAbTV0IX/th+vFnSR0a4fTF68jlHk6m+hfOtZrTT3DnZbuzujROfI6tknfwgEflJdsk+OSYtwskd+UvuyUPlMagHy8HKszWojP8skVcVNJ4AZ0C8JA==</latexit> N2 <latexit sha1_base64="2N2HhVHrFmNqTj/W7r8zUXVj4gk=">AAACWHicfVBdSxtREJ2sbaOxtkYffbk0CKVI2BWxvgiCffCl1IIxghtl9mYSL7kfy713xbDkf/ha/5X+Gu/GCDWKAwOHc84wMyfLpXA+ju9r0cKHj5/qi0uN5c8rX76uNtdOnSkspw430tizDB1JoanjhZd0lltClUnqZqPDSu9ek3XC6BM/zqmncKjFQHD0gbpITRCrWda92N6/XG3F7Xha7DVIZqAFszq+bNZ20r7hhSLtuUTnzpM4970SrRdc0qSRFo5y5CMc0nmAGhW5Xjk9e8I2A9NnA2NDa8+m7P8TJSrnxioLToX+ys1rFfmmlqm5zX6w1yuFzgtPmj8tHhSSecOqTFhfWOJejgNAbkW4nfErtMh9SK7RSH9ReM7S77DoT04WvbE/yhTtUAk9Cc8O060KvWfEm2djQCHnZD7V1+B0u53stnf/7rQODmeJL8IGfIPvkMBPOIAjOIYOcLBwC//grvYQQVSPlp6sUW02sw4vKlp7BBtXtYM=</latexit> <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> W = 2 X 6=↵ b ha↵ , b i N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit> W = N2 y <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> N2 <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W む <latexit sha1_base64="cZg78mnPPwKJ3E/xzhV3bFjVIFE=">AAAC6XicfVFda9RAFJ2kftT1a6uPvgwuQqt1SWKpvhQKVfBFreC2hc5uuJm9yQ47mYSZibiE/AjfRPDJX+W/cbIbod0WLwwczrkfc89NSimMDYI/nr9x4+at25t3enfv3X/wsL/16MQUleY44oUs9FkCBqVQOLLCSjwrNUKeSDxN5ketfvoVtRGF+mIXJY5zyJRIBQfrqLj/6x09oCzVwMM6Yhaqhpkqj2sGspzBQdhM6o9x2GybeMXQl5RJUJlECpMVtZt0gOmlsDOJ6IvLPenFpkwhS9BCc22nVukaTaK4PwiGwTLoVRB2YEC6OI63vD02LXiVo7JcgjHnYVDacQ3aCi6x6bHKYAl8DhmeO6ggRzOulz429JljpjQttHvK0iV7saKG3JhFnrjMHOzMrGstea2W5GuTbfpmXAtVVhYVXw1OK0ltQdsj0anQyK1cOABcC/d3ymfgDLXulL0ee4tuOY0f3KBPJWqwhX7u3NVZLlTjls3Ybov+lwjf/iU65HwO1129Ck6iYbg/3P+8Nzg86hzfJE/IU7JNQvKaHJL35JiMCPc2vB0v8l75c/+7/8P/uUr1va7mMbkU/u+/CUjqVQ==</latexit> E= N1 X 1 2⌧ ↵=1 (s↵ X 1 ↵ ↵ 2 ↵ 2 ha , b i) + ha , b i 2⌧ 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> ↵6= sα 見 4 1 行 目 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 目 ] 2 <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> 8 2 📄 [Saxe-McClelland-Ganguli N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
  23. 1 [Saxe-McClelland-Ganguli ] 2 d ↵ ⌧ a = (s↵

    dt X <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ha↵ , b↵ i)b↵ 6=↵ d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> b ha↵ , b i ha↵ , b↵ i)a↵ X 6=↵ b ha↵ , b i む <latexit sha1_base64="hl5vUVhi53mZqpAaL6x/cLPiFWA=">AAACinicfVBdSxtBFJ2s1dr40dg+9mVoKKhI2BWxSmkR9KEvpQpGBSeGu5ObzeDM7HbmrjQs+Tf9NX1tX/pvOvkoaJReGDiccy5n7kkLrTzF8Z9atPBscen58ov6yura+svGxqsLn5dOYlvmOndXKXjUymKbFGm8KhyCSTVeprfHY/3yDp1XuT2nYYEdA5lVfSWBAtVtfBIabKaRw40AXQxgh6c3IkUCLtxU+chjLr6V0OObUwsXFvnEs9VtNONWPBn+GCQz0GSzOe1u1PZEL5elQUtSg/fXSVxQpwJHSmoc1UXpsQB5CxleB2jBoO9Uk0NH/F1geryfu/As8Ql7f6MC4/3QpMFpgAZ+XhuTT2qpmUum/kGnUrYoCa2cBvdLzSnn4xZ5TzmUpIcBgHQq/J3LATiQFLqu18UJhuMcfglBXwt0QLnbrgS4zCg7CsdmYmeM/meE7/+MAYWek/lWH4OL3Vay39o/22seHc8aX2Zv2Fu2yRL2nh2xz+yUtZlkP9hP9ov9jlaj3egw+jC1RrXZzmv2YKKTv43gxqg=</latexit> ↵ ha , b i = 0 (↵ 6= ) ↵ ↵ ↵ ↵ ha , b i = s↵ or a = b = 0 <latexit sha1_base64="Ea5kdNpN4nwsHMUFsRxlEiHjU5c=">AAACn3icfZFdaxNBFIYn61eNX2l76c1oEERK2C2lelMotFBv1IimrXRjODs52Qydj2XmrDQsyf/qT/HKW/0XTpKtH6l4YOHhPe/hzHk3K5T0FMdfG9GNm7du31m727x3/8HDR631jWNvSyewJ6yy7jQDj0oa7JEkhaeFQ9CZwpPs/GDeP/mCzktrPtKkwL6G3MiRFEBBGrQ+pApMrpDD5xRUMYYtntXEU7ds7XE/uJIIL6iazaybzaa/ZoIj+43xoNWOO/Gi+HVIamizurqD9cZOOrSi1GhIKPD+LIkL6lfgSAqF02ZaeixAnEOOZwENaPT9anH9lD8LypCPrAufIb5Q/5yoQHs/0VlwaqCxX+3NxX/2Mr2ymUav+pU0RUloxHLxqFScLJ9Hy4fSoSA1CQDCyfB2LsbgQFD4Ac1meojhOIdvwqJ3BTog615UKbhcSzMNx+bp1pz+Z4SLK2OgkHOymup1ON7uJLud3fc77f2DOvE19pg9Zc9Zwl6yffaadVmPCXbJvrHv7Ef0JDqK3kbdpTVq1DOb7K+KPv0Ec+vQJg==</latexit> x <latexit sha1_base64="4TKeSCbBf6RPl17MrVhAL1AoEqc=">AAACjXicfVDRahNBFJ1stdaobVoffRkMgkgJu6W2fWilUkFfxAqmLXTS5e7kZjN0ZnaZuSuGJd/j1/iq+DdOshE0Lb0wcO4553LnnqzUylMc/25FK/furz5Ye9h+9PjJ+kZnc+vMF5WT2JeFLtxFBh61stgnRRovSodgMo3n2fXJTD//is6rwn6hSYkDA7lVIyWBApV23goNNtfI4UqALsewzbMrkSEBF65RjrhPG42LIWqCtG7auW2adrpxL54XvwmSBeiyRZ2mm61dMSxkZdCS1OD9ZRKXNKjBkZIap21ReSxBXkOOlwFaMOgH9fzWKX8RmCEfFS48S3zO/jtRg/F+YrLgNEBjv6zNyFu1zCxtptHBoFa2rAitbBaPKs2p4LMg+VA5lKQnAYB0KvydyzE4kBTibrfFOwzHOfwYFn0q0QEV7lUIzuVG2Wk4NhfbM3SXEb79NQYUck6WU70JznZ6yV5v7/Nu9/hkkfgae8aes5csYfvsmH1gp6zPJPvOfrCf7Fe0Eb2ODqM3jTVqLWaesv8qev8HxqXJoA==</latexit> ↵ N2 <latexit sha1_base64="LEHExU3FME6f2R8noD0EWlfCYRM=">AAACi3icfZDfahNBFMYna9Uarab20pvBIIhI2G1LldJioRZ6Y61gmkI3Wc5OTpKh84+Z2WJY9nH6NN7qRd+mk2QLmooHBn585zucOV9uBHc+jm8a0YOVh48erz5pPn229vxFa/3lmdOFZdhlWmh7noNDwRV2PfcCz41FkLnAXn55OOv3rtA6rtV3PzXYlzBWfMQZ+CBlrU+9wSbtDRK6T1NXyKxMQZgJ7CfVoDzJkoq6bKHQ4g6uahikXpus1Y478bzofUhqaJO6TrP1xnY61KyQqDwT4NxFEhvfL8F6zgRWzbRwaIBdwhgvAiqQ6Prl/NKKvgnKkI60DU95Olf/nChBOjeVeXBK8BO33JuJ/+zlcmmzH33sl1yZwqNii8WjQlCv6SxGOuQWmRfTAMAsD3+nbAIWmA9hN5vpZwzHWfwSFn01aMFr+y5ka8eSqyocO07fz+h/RvhxZwwUck6WU70PZ5udZKez8227fXBYJ75KXpHX5C1JyAdyQI7JKekSRq7JT/KL/I7Woq1oN9pbWKNGPbNB/qro6BZtgcgf</latexit> 2 1 W W = N1 X > s ↵ u↵ v ↵ <latexit sha1_base64="haSdbYtsNQyoRe7ykMC/yp4rtBI=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hordCtMptO19AkuyRZsSz9CV71Z3n2h3gTMa0VtBUHAg/vvGFm3jDhTBvPe3VyU9Mzs/m5+cLC4tLyympx7UrHqaJYpzGP1XUIGjmTWDfMcLxOFIIIOTbCbm3Qb9yj0iyWl6aXYEtAJFmHUTBWumjc+LerJa/sDcudBH8EJTKqs9uiUwnaMU0FSkM5aN30vcS0MlCGUY79QpBqTIB2IcKmRQkCdSsb7tp3t6zSdjuxsk8ad6j+/JGB0LonQusUYO70eG8g/tkLxdhk0zlsZUwmqUFJvwZ3Uu6a2B0E4baZQmp4zwJQxezuLr0DBdTYuAqF4AjtcQpP7KDTBBWYWO1kAahIMNm3x0bB7oD+M8LDt9GSzdkfT3USrvbK/n55/7xSqtZGic+RDbJJtolPDkiVHJMzUieUROSRPJFn58V5c96djy9rzhn9WSe/Kpf/BPCDs/Q=</latexit> W ha , b i = s↵ ↵ む y <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> <latexit sha1_base64="NgEKFcYW2o1IZfa5uw8QAVDCf3s=">AAACTXicfZBNS8NAEIY39aNav1o9egkWQURKUkQ9FvTgxS+0KphSJttpXLq7CbsbsYT+BK/6szz7Q7yJuK0RtIoDCw/vvMvMvGHCmTae9+IUJianposzs6W5+YXFpXJl+VLHqaLYpDGP1XUIGjmT2DTMcLxOFIIIOV6Fvf1h/+oOlWaxvDD9BFsCIsm6jIKx0vlxu94uV72aNyr3N/g5VElep+2Ksx10YpoKlIZy0PrG9xLTykAZRjkOSkGqMQHagwhvLEoQqFvZaNeBu26VjtuNlX3SuCP1+48MhNZ9EVqnAHOrx3tD8c9eKMYmm+5eK2MySQ1K+jm4m3LXxO4wCLfDFFLD+xaAKmZ3d+ktKKDGxlUqBQdoj1N4ZAedJKjAxGozC0BFgsmBPTYKtob0nxHuv4yWbM7+eKq/4bJe83dqO2fb1cZ+nvgMWSVrZIP4ZJc0yCE5JU1CSUQeyCN5cp6dV+fNef+0Fpz8zwr5UYXiB+Mhs+0=</latexit> <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> 1 <latexit sha1_base64="aufhfgnOg5Qu4gUpqbQ+th/Tt6I=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1WNCDF1HR2kK3ltl0ug0m2SXJimXpT/CqP8uzP8SbiGmtoFUcCDy884aZecOEM20878XJTU3PzObn5gsLi0vLK8XS6rWOU0WxRmMeq0YIGjmTWDPMcGwkCkGEHOvh7dGwX79DpVksr0w/wZaASLIuo2CsdFm/2W0Xy17FG5X7G/wxlMm4ztslZy/oxDQVKA3loHXT9xLTykAZRjkOCkGqMQF6CxE2LUoQqFvZaNeBu2mVjtuNlX3SuCP1+48MhNZ9EVqnANPTk72h+GcvFBOTTfewlTGZpAYl/RzcTblrYncYhNthCqnhfQtAFbO7u7QHCqixcRUKwTHa4xSe2kFnCSowsdrOAlCRYHJgj42CnSH9Z4T7L6Mlm7M/mepvuN6t+PuV/Yu9cvVonPgcWScbZIv45IBUyQk5JzVCSUQeyCN5cp6dV+fNef+05pzxnzXyo3L5D/Jqs/U=</latexit> W 2 N1 <latexit sha1_base64="rULBMhMqQ9482yCyBwTKgM//9KA=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi19otdAtZTadrqFJdkmyYln6E7zqz/LsD/EmYloraBUHAg/vvGFm3jDhTBvPe3FyU9Mzs/m5+cLC4tLyympx7VrHqaJYpzGPVSMEjZxJrBtmODYShSBCjjdhrzbs39yh0iyWV6afYEtAJFmXUTBWujxt++3Vklf2RuX+Bn8MJTKu83bRqQSdmKYCpaEctG76XmJaGSjDKMdBIUg1JkB7EGHTogSBupWNdh24W1bpuN1Y2SeNO1K//8hAaN0XoXUKMLd6sjcU/+yFYmKy6R62MiaT1KCkn4O7KXdN7A6DcDtMITW8bwGoYnZ3l96CAmpsXIVCcIT2OIUndtBZggpMrHayAFQkmBzYY6Ngd0j/GeH+y2jJ5uxPpvobrvfK/n55/6JSqtbGic+RDbJJtolPDkiVHJNzUieUROSBPJIn59l5dd6c909rzhn/WSc/Kpf/AOE6s+w=</latexit> ↵=1 大 4 1 行 十 目 目 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 9 2 📄 saddle N3 <latexit sha1_base64="EMusQFr7bl0LES0y9LepKT4756Y=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRX1KOjBi19oq9AtZTadbkOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkJianpvMzs4W5+YXFpWJpuabjVFGs0pjH6jYEjZxJrBpmON4mCkGEHG/C7tGgf3OHSrNYXptegg0BkWRtRsFY6eqsudMslr2KNyz3N/gjKJNRXTRLzm7QimkqUBrKQeu67yWmkYEyjHLsF4JUYwK0CxHWLUoQqBvZcNe+u26VltuOlX3SuEP1+48MhNY9EVqnANPR472B+GcvFGOTTfugkTGZpAYl/RzcTrlrYncQhNtiCqnhPQtAFbO7u7QDCqixcRUKwTHa4xSe2kHnCSowsdrMAlCRYLJvj42CrQH9Z4T7L6Mlm7M/nupvqG1X/L3K3uVu+fBolPgMWSVrZIP4ZJ8ckhNyQaqEkog8kEfy5Dw7r86b8/5pzTmjPyvkR+XyH+UIs+4=</latexit>
  24. [Saxe-McClelland-Ganguli 1 2 d ↵ ⌧ a = (s↵ dt

    X <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ha↵ , b↵ i)b↵ 6=↵ d ↵ ⌧ b = (s↵ dt <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> b ha↵ , b i : a (0) = a↵ (0)r , b (0) = b↵ (0)r ↵ <latexit sha1_base64="Nf2/J4Ynmmon5e39QAN5JUha0Ko=">AAACkXicfVBdSxtBFJ2s/dDYj0QffRkaBFsk7EqI+iAEoyCUooVGA9k03J3cJENmZpeZWTEs+UX+mj5W/4yTZEWNpRcGzj3nXO7cEyWCG+v7fwveypu3796vrhXXP3z89LlU3rg0caoZtlgsYt2OwKDgCluWW4HtRCPISOBVNG7O9Ktr1IbH6pedJNiVMFR8wBlYR/VKp/A7BJGMYMf/So8o9J46nSu7NHrhiZ48j5ZeqeJX/XnR1yDIQYXkddErF2phP2apRGWZAGM6gZ/YbgbaciZwWgxTgwmwMQyx46ACiaabze+d0m3H9Okg1u4pS+fs84kMpDETGTmnBDsyy9qM/KcWyaXNdnDQzbhKUouKLRYPUkFtTGdh0j7XyKyYOABMc/d3ykaggVkXebEYnqA7TuMPt+g8QQ021t+yEPRQcjV1xw7D3Rn6nxFuHo0OuZyD5VRfg8u9alCv1n/WKo1mnvgq2SJfyA4JyD5pkDNyQVqEkVvyh9yRe2/TO/Qa3vHC6hXymU3yorzvD1CnyC0=</latexit> d ↵ ⌧ a = (s↵ dt ↵ X <latexit sha1_base64="AfJpY4yJMF2bPyXsWRiLbIFyJ0o=">AAAC5nicfVFNaxRBEO0Zv+L6tYlHL42LEiVuZiQkXoRAPHgRI7hJIL0ONb01s026e4buHnEZ5i94E8GTf8t/Y8/sLOomWNDweO9VVVdVWkphXRT9CsJr12/cvLVxe3Dn7r37D4abWye2qAzHCS9kYc5SsCiFxokTTuJZaRBUKvE0vThq9dPPaKwo9Ee3KHGqINciExycp5LhD+agoiwzwOtZU89cA58YyHIO9DXdtkmPX1AmQecS6UreoenKyEwnPfvDeLutVFKzHJQCppEuhaa1dNzV9XppWS8ZjqJx1AW9DOIejEgfx8lmsMdmBa8UasclWHseR6Wb1mCc4BKbAasslsAvIMdzDzUotNO6W2JDn3hmRrPC+Kcd7di/M2pQ1i5U6p0K3Nyuay15pZaqtc4uezWthS4rh5ovG2eVpK6g7YXoTBjkTi48AG6E/zvlc/AHcv6OgwF7g344g+98o/clGnCFeV4zMLkSuvHD5mynRf8zwpeV0SO/53h9q5fByctxvD/e/7A3OjzqN75BHpHHZJvE5IAckrfkmEwID0jwNNgNonAefg2/hd+X1jDocx6SfyL8+RshW+iw</latexit> ha↵ , b↵ i)b↵ 6=↵ ↵ ↵ <latexit sha1_base64="ekaTd1bJ58JDE7/+e9JBZ1zDKbI=">AAAChHicfZBdSxtBFIYna/1orDbay94MDYKUEHatXzeKoBe9ES00KrgxnJ2cbAZnZpeZs2JY8lf8Nb1t7/03nU1SaGPxwMDDed/DmfMmuZKOwvC5Fiy8WVxaXnlbX323tv6+sbF55bLCCuyITGX2JgGHShrskCSFN7lF0InC6+T+tNKvH9A6mZnvNMqxqyE1ciAFkG/1GoexApMq5PYuBpUPoVVRggQ8tlPliMd9VAS9cuqYqONeoxm2w0nxlxDNoMlmddnbqO3G/UwUGg0JBc7dRmFO3RIsSaFwXI8LhzmIe0jx1qMBja5bTk4c8y3f6fNBZv0zxCfdvydK0M6NdOKdGmjo5rWq+V8t0XObaXDYLaXJC0IjposHheKU8So/3pcWBamRBxBW+r9zMQQLgnzK9Xp8hv44i+d+0UWOFiizn31wNtXSjP2xadyq6DUjPP4xevI5R/OpvoSrnXa0397/tts8OZ0lvsI+sk9sm0XsgJ2wr+ySdZhgT+wH+8l+BUtBK/gS7E2tQW0284H9U8Hxb58mxhM=</latexit> ha↵ , b↵ i)a↵ ↵ hr , r i = d ↵ ⌧ b = (s↵ dt b ha↵ , b i X b ha↵ , b i X b ha↵ , b i 6=↵ ↵ <latexit sha1_base64="PJ2wG72ScPWOCIlChp32K9sl1Os=">AAAC5nicfVFda9RAFJ3Er7p+bfXRl8FFqVK3iZTqi1CoD76IFdy20FnDzeQmO3RmEmYm4hLyF3wTwSf/lv/GSTYW3RYvDBzOOffeufemlRTWRdGvILxy9dr1Gxs3R7du37l7b7x5/8iWteE446UszUkKFqXQOHPCSTypDIJKJR6nZwedfvwZjRWl/uiWFc4VFFrkgoPzVDL+wRzUlOUGeJO1Teba9BMDWS2AvqZbNhnwc8ok6EIihUHepudGZnrp6bnW2W2tkoYVoBQwjXQltF1Sz11eb5BW9ZLxJJpGfdCLIB7AhAxxmGwGuywrea1QOy7B2tM4qty8AeMEl9iOWG2xAn4GBZ56qEGhnTf9Elv62DMZzUvjn3a0Z//OaEBZu1SpdypwC7uudeSlWqrWOrv81bwRuqodar5qnNeSupJ2F6KZMMidXHoA3Aj/d8oX4A/k/B1HI/YG/XAG3/lG7ys04ErzrGFgCiV064ct2HaH/meEL3+MHvk9x+tbvQiOXkzjveneh93J/sGw8Q3ykDwiWyQmL8k+eUsOyYzwgARPgp0gChfh1/Bb+H1lDYMh5wH5J8KfvwEhluiw</latexit> ha↵ , b↵ i)a↵ 6=↵ decoupled d ⌧ a = b(s dt d ⌧ b = a(s dt <latexit sha1_base64="e30gZbrWQ0ow6F5zp0ZEP3MnZhU=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqoF/58mm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqG7Gg==</latexit> <latexit sha1_base64="n5jjzTsvJdOimLjKJySjsUXqeSM=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqhP+ncOm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqW7Gg==</latexit> ab) 1 <latexit sha1_base64="bDK/i3gH15rrQq6tjoo/A6Erpbc=">AAACgnicfVBdSxtBFJ1sbbWpbWN97MtgKIhI2E0lFlpBsGBfihaaRHBjuDu5iYPzsczcLYYl/8Rf09f2D/TfdDZG0Fh6YeBwzrn3zj1ZrqSnOP5Ti56sPH22uva8/mL95avXjY03PW8LJ7ArrLLuLAOPShrskiSFZ7lD0JnCfnZ1VOn9H+i8tOY7TXMcaJgYOZYCKFDDRqd/kfADfpza4KqG8Iro7fL0Y0DtIHXvS21+fJGSzYeNZtyK58Ufg2QBmmxRp8ON2l46sqLQaEgo8P48iXMalOBICoWzelp4zEFcwQTPAzSg0Q/K+YEz/i4wIz62LjxDfM7e7yhBez/VWXBqoEu/rFXkP7VML22m8YdBKU1eEBpxu3hcKE6WV+nxkXQoSE0DAOFk+DsXl+BAUMi4Xk8/YzjO4dew6CRHB2TdTpmCm2hpZuHYSbpbof8Z4frOGFDIOVlO9THotVtJp9X5ttc8PFokvsbesi22zRK2zw7ZF3bKukywG/aT/WK/o5VoJ0qi97fWqLbo2WQPKvr0F3Izwh4=</latexit> 1 む 2 2 W = GW V, W = U W G 2 <latexit sha1_base64="lZRa2sOHegu1hDN7u2W7PW3biFw=">AAACeHicfVBdSxtBFJ2s/dD0K9ZHX4aGUi0SdoNYXwqCPvSlaKFJBDcJdyc3cXA+lpm7xbDkT/hrfNV/4V/pU2djBBtLLwycOffcOXNPlivpKY7vatHKs+cvXq6u1V+9fvP2XWP9fdfbwgnsCKusO83Ao5IGOyRJ4WnuEHSmsJddHFb93i90Xlrzk6Y59jVMjBxLARSoYWOnN2j3Bgn/yjtbqQ3K6iEeyMeXZLs7SMnmw0YzbsXz4k9BsgBNtqiT4XptNx1ZUWg0JBR4f5bEOfVLcCSFwlk9LTzmIC5ggmcBGtDo++V8rRn/GJgRH1sXjiE+Zx9PlKC9n+osKDXQuV/uVeQ/e5lecqbxfr+UJi8Ijbg3HheKk+VVZnwkHQpS0wBAOBn+zsU5OBAUkq3X0yMMyzn8HoyOc3RA1n0uU3ATLc0sLDtJdyr0PyFcPggDCjkny6k+Bd12K9lr7f3YbR4cLhJfZZvsA9tiCfvCDtg3dsI6TLArds1u2G3td8SjT9H2vTSqLWY22F8Vtf8A15HAvA==</latexit> 1 ; W 2 W 1 = U (W W )V > ab) (G 2 R > <latexit sha1_base64="1s/ZqHsgtJaGz/ba32ItsUZHe4I=">AAACa3icfZDfShtBFMYnq2019k/UOxUcDIKUEnZFbC8FBXvjX4wKbgxnJydxcGZ2mTlbGpa99Gl6Wx+mD+E7OBsjtLF4YODHd87wnfMlmZKOwvBPLZiafvP23cxsfe79h4+fGvML5y7NrcC2SFVqLxNwqKTBNklSeJlZBJ0ovEhud6v+xQ+0TqbmjIYZdjQMjOxLAeSlbmN1P5Ym1kA3SVKcltfFYXeTxyQ1Ou6x7DaaYSscFX8J0RiabFzH3fnaVtxLRa7RkFDg3FUUZtQpwJIUCst6nDvMQNzCAK88GvBWnWJ0ScnXvdLj/dT6Z4iP1L9/FKCdG+rET1Y7u8leJf63l+gJZ+p/6xTSZDmhEU/G/VxxSnkVE+9Ji4LU0AMIK/3uXNyABUE+zHo93kN/nMUDb3SUoQVK7eciBjvQ0pT+2EH8paLXBuHn86Ann3M0mepLON9sRdut7ZOt5s7uOPEZtszW2AaL2Fe2w76zY9Zmgt2xX+w3u689BIvBUrDyNBrUxn8W2T8VrD8Cwy28ug==</latexit> N2 ⇥N2 : ) ⌃ = U SV > <latexit sha1_base64="gqsoMc39D4bwJRNI9IXCMxyR9zQ=">AAACXXicfZDfShtBFMYna602tTaxF73wZjAUpJSwW0S9EQR74U2pYhMFN4azk5N1cP4sM2fVsORRvNVn8qqv0klMoY3FAwM/vvMN55wvK5T0FMePtWjh1eLrpeU39bcr71bfN5prXW9LJ7AjrLLuLAOPShrskCSFZ4VD0JnC0+zqYNI/vUbnpTU/aVRgT0Nu5FAKoCD1G830ROYa+B7vnHQvUrJFv9GK2/G0+HNIZtBiszrqN2tb6cCKUqMhocD78yQuqFeBIykUjutp6bEAcQU5ngc0oNH3qunuY/4pKAM+tC48Q3yq/v2jAu39SGfBqYEu/XxvIv63l+m5yTTc7VXSFCWhEU+Dh6XiZPkkGD6QDgWpUQAQTobdubgEB4JCfPV6+g3DcQ6/h0E/CnRA1n2uUnC5lmYcjs3TLxN6yQi3f4yBQs7JfKrPofu1nWy3t4+3WvsHs8SX2TrbYJssYTtsnx2yI9Zhgt2wO3bPHmq/osVoJVp9ska12Z8P7J+KPv4Gphi2ow==</latexit> 行 力 入 4 1 行 目 目 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 0 3 📄 : ]
  25. 1 2 d ⌧ a = b(s dt d ⌧

    b = a(s dt <latexit sha1_base64="e30gZbrWQ0ow6F5zp0ZEP3MnZhU=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqoF/58mm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqG7Gg==</latexit> <latexit sha1_base64="n5jjzTsvJdOimLjKJySjsUXqeSM=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqhP+ncOm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqW7Gg==</latexit> ab) 1 E(a, b) = 2⌧ (s む <latexit sha1_base64="GLKH/1hpZny1m7QVE1xHGjqHC00=">AAACdXicfZDRShtBFIYna7U21Tbay1IYTJWkpOluEO1NQbCF3pRaaFRw03B2cjYOzswuM2fFsOQZ+jTe6nP4JL3tJKagsXhg4OM//+HM+ZNcSUdheFMJFp4sLj1dflZ9vrL64mVtbf3QZYUV2BWZyuxxAg6VNNglSQqPc4ugE4VHydn+pH90jtbJzPykUY49DUMjUymAvNSvNb80oJU0+SceE16Qo5FCHqcWRFR2YoJi3HDvIWn+6vRr9bAdTos/hGgGdTarg/5aZTseZKLQaEgocO4kCnPqlWBJCoXjalw4zEGcwRBPPBrQ6Hrl9KYx3/TKgKeZ9c8Qn6p3J0rQzo104p0a6NTN9ybif3uJnttM6cdeKU1eEBpxuzgtFKeMTwLjA2lRkBp5AGGl/zsXp+ADIh9rtRp/Rn+cxW9+0fccLVBm35Ux2KGWZuyPHcatCT1mhIt/Rk8+52g+1Ydw2GlHO+2dH9v1vf1Z4svsNdtgDRaxXbbHvrID1mWC/WaX7IpdV/4Eb4K3wdatNajMZl6xexV8+Av/Cr9v</latexit> s = ab 2 a <latexit sha1_base64="zxeL66RFJEXuUZn7DMsZnD7/Hdk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQNd5slr+yNy/0N/hRKZFpXd0WnEnQimgiUhnLQuuV7sWmnoAyjHEeFINEYAx1AD1sWJQjU7XS86cjds0rH7UbKPmncsfr9RwpC66EIrVOA6evZXib+2QvFzGTTPW2nTMaJQUkng7sJd03kZjG4HaaQGj60AFQxu7tL+6CAGhtWoRCcoT1O4YUddBmjAhOpgzQA1RNMjuyxveAwo/+M8PBltGRz9mdT/Q2No7J/XD6+rpSqtWniebJDdsk+8ckJqZJzckXqhBIkj+SJPDsvzpvz7nxMrDln+meb/Kjcwien1bNb</latexit> = b ab) ab) s = ab む <latexit sha1_base64="PW3tAs1QCudVRa9RYJQKp+evktU=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bJbRD0KevAiKlgruFVm02kNTbJLkhXL0h/hVX+WJ3+KN9NaQas4EHh45w0z8yaZ4MaG4Zvnj41PTE5Nz5Rm5+YXFpeWV65MmmtGNZaKVF8naEhwRTXLraDrTBPKRFA96Rz1+/UH0oan6tJ2M2pIbCve4gytk+p4W91Jbqt3S+WwEg4q+A3REMowrPO7ZW83bqYsl6QsE2jMTRRmtlGgtpwJ6pXi3FCGrINtunGoUJJpFIN9e8GGU5pBK9XuKRsM1O8/CpTGdGXinBLtvRnt9cU/e4kcmWxbB42Cqyy3pNjn4FYuApsG/TCCJtfErOg6QKa52z1g96iRWRdZqRQfkztO06kbdJaRRpvqrSJG3ZZc9dyx7Xi7T/8Z8fHL6MjlHI2m+huuqpVor7J3sVs+PBomPg1rsA6bEME+HMIJnEMNGHTgCZ7hxXv13n3w/U+r7w3/rMKP8mc/AH6Vs0g=</latexit> a 2 b a <latexit sha1_base64="i3ODyEEUl87ObflhTOKxPnEzK24=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsiqgXoVAPXsQKVgvdIrPptA0m2SXJimXpX/CqP8urf8SbaLauoFUcCDy884aZecOYM20878UpTE3PzM4V50sLi0vLK6vltSsdJYpik0Y8Uq0QNHImsWmY4diKFYIIOV6Ht/Wsf32HSrNIXpphjB0Bfcl6jILJJH0M4c1qxat643J/g59DheTVuCk7+0E3oolAaSgHrdu+F5tOCsowynFUChKNMdBb6GPbogSBupOOlx25W1bpur1I2SeNO1a//0hBaD0UoXUKMAM92cvEP3uhmJhsekedlMk4MSjp5+Bewl0TuVkSbpcppIYPLQBVzO7u0gEooMbmVSoFJ2iPU3hmB53HqMBEaicNQPUFkyN7bD/Yzeg/I9x/GS3ZnP3JVH/D1V7VP6geXOxXavU88SLZIJtkm/jkkNTIKWmQJqFkQB7II3lynp1X5815/7QWnPzPOvlRheIHI0+0iw==</latexit> 2 <latexit sha1_base64="Rhwfy4KFyngwypMReK/iGk+an1E=">AAACh3icfZDbahRBEIZ7x1OyHrLRS28aF0FE15kQ1kAQIvHCGzGCmw1sT5aenprZJn0YumuCyzAP49N4q5e+jb2bEXQjFjR89VcV1fVnlZIe4/hnL7px89btO1vb/bv37j/YGew+PPW2dgImwirrzjLuQUkDE5So4KxywHWmYJpdHK/q00twXlrzGZcVpJqXRhZScAzSfHDICsdFk7dNji1TUOBsep404bXnDG1FX9KQ7HVJIOZkucCUvqHxfDCMR/E66HVIOhiSLk7mu719lltRazAoFPd+lsQVpg13KIWCts9qDxUXF7yEWUDDNfi0WV/Z0qdByWlhXXgG6Vr9c6Lh2vulzkKn5rjwm7WV+M9apjc2Y3GQNtJUNYIRV4uLWlG0dGUhzaUDgWoZgAsnw9+pWPBgIwaj+332DsJxDj6ERR8rcByte94w7kotTRuOLdmLFf2vkX/53Rgo+JxsunodTvdGyXg0/rQ/PDruHN8ij8kT8owk5DU5Iu/JCZkQQb6Sb+Q7+RFtR6+icXRw1Rr1uplH5K+I3v4CXQ3Grg==</latexit> <latexit sha1_base64="C7WIyqWowAWoqVB4i9nLF/HUqIg=">AAACfHicfZBdaxNBFIYn61eNX6leejMYhfqRsLuU2Buh0F54I1YwbaGbhjOzZ9OhM7PLzFlpWPZv+Gu81f/gn5FO0giaigcGHt7zHs6cV1RaeYrjn53oxs1bt+9s3O3eu//g4aPe5uNDX9ZO4liWunTHAjxqZXFMijQeVw7BCI1H4nxv0T/6gs6r0n6meYUTAzOrCiWBgjTtxRkVDmSTt01O7RacpgNxmr7k73gKWV4SBz7gqViiCGo87fXjYbwsfh2SFfTZqg6mm53tMC1rg5akBu9PkriiSQOOlNTYdrPaYwXyHGZ4EtCCQT9plqe1/EVQcl6ULjxLfKn+OdGA8X5uRHAaoDO/3luI/+wJs7aZip1Jo2xVE1p5tbioNaeSL3LjuXIoSc8DgHQq/J3LMwjRUUi32832MRzn8ENY9LFCB1S6V00GbmaUbcOxs+zNgv5nhIvfxkAh52Q91etwmA6T0XD0abu/u7dKfIM9Zc/YFkvYW7bL3rMDNmaSfWXf2Hf2o/Mreh69jgZX1qizmnnC/qpodAn+h8EE</latexit> 2 d dt (a 2 b ) = 2aȧ 2bḃ = 0 d h 1 1> W W dt matrix balancing W 2> i W2 = 0 d ⌧ u = 2u(s dt <latexit sha1_base64="MnNkMwQbfToD1Bjp1o6pBovfJjY=">AAACZ3icfZDPShxBEMZ7R5OYzR9XAyJ46WQJmJAsMyImF0HQgxfRQFYFZ1lqemrWxu6eobs6ZBnmkKfJVR8nj+Bb2LtuIFlDChp+fPUV1fVllZKO4vhXK1pYfPT4ydLT9rPnL14ud1ZWT13prcC+KFVpzzNwqKTBPklSeF5ZBJ0pPMuu9if9s29onSzNVxpXONAwMrKQAihIw85GSuDTwoKo86bOqfF8l2/5TffRvxt2unEvnhZ/CMkMumxWJ8OV1naal8JrNCQUOHeRxBUNarAkhcKmnXqHFYgrGOFFQAMa3aCeXtHwt0HJeVHa8AzxqfrnRA3aubHOglMDXbr53kT8Zy/Tc5up+Dyopak8oRH3iwuvOJV8EhHPpUVBahwAhJXh71xcQoiIQpDtdnqA4TiLR2HRcYUWqLTv6xTsSEvThGNH6YcJ/c8I338bA4Wck/lUH8LpVi/Z6e182e7u7c8SX2Ib7A3bZAn7xPbYITthfSbYD/aTXbOb1m20HK1F6/fWqDWbecX+quj1HQEIuvs=</latexit> むa = b u ⌘ ab <latexit sha1_base64="9zU9Q5ZXCuYTOa3DLx3d5fhQbQ8=">AAACTXicfZBNSwMxEIaz9aO1flU9elksgoiUXRH1Igj14EVUtFboljKbTtfQJLskWbEs/Qle9Wd59od4EzGtFbQVBwIP77xhZt4w4Uwbz3t1clPTM7P5wlxxfmFxabm0snqj41RRrNGYx+o2BI2cSawZZjjeJgpBhBzrYbc66NfvUWkWy2vTS7ApIJKswygYK13BUdgqlb2KNyx3EvwRlMmoLlorzl7QjmkqUBrKQeuG7yWmmYEyjHLsF4NUYwK0CxE2LEoQqJvZcNe+u2mVttuJlX3SuEP1548MhNY9EVqnAHOnx3sD8c9eKMYmm85hM2MySQ1K+jW4k3LXxO4gCLfNFFLDexaAKmZ3d+kdKKDGxlUsBidoj1N4ZgedJ6jAxGo7C0BFgsm+PTYKdgb0nxEevo2WbM7+eKqTcLNb8fcr+5d75ePqKPECWScbZIv45IAck1NyQWqEkog8kify7Lw4b8678/FlzTmjP2vkV+XynyH7tA4=</latexit> <latexit sha1_base64="QqO10Uz6UXy375kIMx9DT0GF/Sk=">AAACVHicfZDPSgMxEMazq/VP1drq0UuwCCJSdkXUo6AHL6KCtaJbZDadtqFJdk2yYln6Fl71sQTfxYNpraBVHAj8+OYLM/PFqeDGBsGb509NF2Zm5+aLC4tLpeVyZeXKJJlmWGeJSPR1DAYFV1i33Aq8TjWCjAU24t7RsN94QG14oi5tP8WmhI7ibc7AOukmi/A+4w8U4rtyNagFo6K/IRxDlYzr/K7i7UathGUSlWUCjLkNg9Q2c9CWM4GDYpQZTIH1oIO3DhVINM18tPKAbjilRduJdk9ZOlK//8hBGtOXsXNKsF0z2RuKf/ZiOTHZtg+aOVdpZlGxz8HtTFCb0GEetMU1Miv6DoBp7nanrAsamHWpFYvRMbrjNJ66QWcparCJ3soj0B3J1cAd24m2h/SfER6/jI5czuFkqr/haqcW7tX2Lnarh0fjxOfIGlknmyQk++SQnJBzUieMKPJEnsmL9+q9+1N+4dPqe+M/q+RH+aUPaXu1NA==</latexit> u(t) = se e2st/⌧ u) Riccati <latexit sha1_base64="Nk9PZApyQlyQJmnxHqZ7ccbUQvI=">AAACinicfZBdSxtBFIYn2w9tamusl70ZDIXYj7gbxFZKi6AXvSm10KjgpuHs5GwcnJldZs6WhmH/TX9Nb+uN/8ZJjGBj6YGBZ97zDmfOm5VKOorjy0Z07/6Dh0vLj5qPV548XW2tPTtyRWUF9kWhCnuSgUMlDfZJksKT0iLoTOFxdr4/7R//QOtkYb7RpMSBhrGRuRRAQRq2PlYd2uQfeJpbEN7hd99ztJUSVHXtb9/4G57wVze+2ledeLOuh6123I1nxe9CMoc2m9fhcK2xnY4KUWk0JBQ4d5rEJQ08WJJCYd1MK4cliHMY42lAAxrdwM8WrfmLoIx4XthwDPGZevuFB+3cRGfBqYHO3GJvKv6zl+mFyZS/G3hpyorQiOvBeaU4FXyaIh9Ji4LUJAAIK8PfuTiDEA2FrJvN9ADDchY/h0FfSrRAhX3pU7BjLU0dlh2nr6f0PyP8vDEGCjkni6nehaNeN9np7nzdbu/tzxNfZs/ZBuuwhL1le+wTO2R9Jtgv9pv9YRfRStSLdqP319aoMX+zzv6q6OAKzuHHTw==</latexit> 2st/⌧ s 1 + u(0) s 一 行 行 方 目 目 一 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 1 一 3 📄 b <latexit sha1_base64="067UFn5yTNcR3dGAVXVkIrV15gk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQd3m2WvLI3Lvc3+FMokWld3RWdStCJaCJQGspB65bvxaadgjKMchwVgkRjDHQAPWxZlCBQt9PxpiN3zyodtxsp+6Rxx+r3HykIrYcitE4Bpq9ne5n4Zy8UM5NN97SdMhknBiWdDO4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYhUJwhvY4hRd20GWMCkykDtIAVE8wObLH9oLDjP4zwsOX0ZLN2Z9N9Tc0jsr+cfn4ulKq1qaJ58kO2SX7xCcnpErOyRWpE0qQPJIn8uy8OG/Ou/Mxseac6Z9t8qNyC5+pvLNc</latexit>
  26. 2 d ⌧ a = b(s dt d ⌧ b

    = a(s dt <latexit sha1_base64="e30gZbrWQ0ow6F5zp0ZEP3MnZhU=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqoF/58mm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqG7Gg==</latexit> <latexit sha1_base64="n5jjzTsvJdOimLjKJySjsUXqeSM=">AAACaHicfZDBShxBEIZ7JybRVZM1HkT00LgENCTLTBDjJSDowYtowFXBWZaanpq1sbtn6K4JWYa5+DRe49v4CnmK9K4r6BpS0PDx119U158USjoKw/tG8Grm9Zu3s3PN+YXFd+9bSx/OXF5agV2Rq9xeJOBQSYNdkqTworAIOlF4nlzvj/rnP9E6mZtTGhbY0zAwMpMCyEv91npMUPI4syCqtK5SqhP+ncOm+wLJVr/VDjvhuPhLiCbQZpM66S81tuM0F6VGQ0KBc5dRWFCvAktSKKybcemwAHENA7z0aECj61XjM2r+0Sspz3LrnyE+Vp9OVKCdG+rEOzXQlZvujcR/9hI9tZmy3V4lTVESGvGwOCsVp5yPMuKptChIDT2AsNL/nYsr8BGRT7LZjA/QH2fxyC86LtAC5fZTFYMdaGlqf+wg/jyi/xnh16PRk885mk71JZx97UQ7nZ0f2+29/Unis2yNbbBNFrFvbI8dshPWZYLdsFv2m901/gStYCVYfbAGjcnMMntWwcZfUqW7Gg==</latexit> ab) 1 E(a, b) = 2⌧ (s む <latexit sha1_base64="GLKH/1hpZny1m7QVE1xHGjqHC00=">AAACdXicfZDRShtBFIYna7U21Tbay1IYTJWkpOluEO1NQbCF3pRaaFRw03B2cjYOzswuM2fFsOQZ+jTe6nP4JL3tJKagsXhg4OM//+HM+ZNcSUdheFMJFp4sLj1dflZ9vrL64mVtbf3QZYUV2BWZyuxxAg6VNNglSQqPc4ugE4VHydn+pH90jtbJzPykUY49DUMjUymAvNSvNb80oJU0+SceE16Qo5FCHqcWRFR2YoJi3HDvIWn+6vRr9bAdTos/hGgGdTarg/5aZTseZKLQaEgocO4kCnPqlWBJCoXjalw4zEGcwRBPPBrQ6Hrl9KYx3/TKgKeZ9c8Qn6p3J0rQzo104p0a6NTN9ybif3uJnttM6cdeKU1eEBpxuzgtFKeMTwLjA2lRkBp5AGGl/zsXp+ADIh9rtRp/Rn+cxW9+0fccLVBm35Ux2KGWZuyPHcatCT1mhIt/Rk8+52g+1Ydw2GlHO+2dH9v1vf1Z4svsNdtgDRaxXbbHvrID1mWC/WaX7IpdV/4Eb4K3wdatNajMZl6xexV8+Av/Cr9v</latexit> ab) ab) s = ab 2 a <latexit sha1_base64="zxeL66RFJEXuUZn7DMsZnD7/Hdk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQNd5slr+yNy/0N/hRKZFpXd0WnEnQimgiUhnLQuuV7sWmnoAyjHEeFINEYAx1AD1sWJQjU7XS86cjds0rH7UbKPmncsfr9RwpC66EIrVOA6evZXib+2QvFzGTTPW2nTMaJQUkng7sJd03kZjG4HaaQGj60AFQxu7tL+6CAGhtWoRCcoT1O4YUddBmjAhOpgzQA1RNMjuyxveAwo/+M8PBltGRz9mdT/Q2No7J/XD6+rpSqtWniebJDdsk+8ckJqZJzckXqhBIkj+SJPDsvzpvz7nxMrDln+meb/Kjcwien1bNb</latexit> s = ab <latexit sha1_base64="i3ODyEEUl87ObflhTOKxPnEzK24=">AAACTnicfZBNSwMxEIaz9avWz+rRy2IRRKTsiqgXoVAPXsQKVgvdIrPptA0m2SXJimXpX/CqP8urf8SbaLauoFUcCDy884aZecOYM20878UpTE3PzM4V50sLi0vLK6vltSsdJYpik0Y8Uq0QNHImsWmY4diKFYIIOV6Ht/Wsf32HSrNIXpphjB0Bfcl6jILJJH0M4c1qxat643J/g59DheTVuCk7+0E3oolAaSgHrdu+F5tOCsowynFUChKNMdBb6GPbogSBupOOlx25W1bpur1I2SeNO1a//0hBaD0UoXUKMAM92cvEP3uhmJhsekedlMk4MSjp5+Bewl0TuVkSbpcppIYPLQBVzO7u0gEooMbmVSoFJ2iPU3hmB53HqMBEaicNQPUFkyN7bD/Yzeg/I9x/GS3ZnP3JVH/D1V7VP6geXOxXavU88SLZIJtkm/jkkNTIKWmQJqFkQB7II3lynp1X5815/7QWnPzPOvlRheIHI0+0iw==</latexit> b 2 <latexit sha1_base64="Rhwfy4KFyngwypMReK/iGk+an1E=">AAACh3icfZDbahRBEIZ7x1OyHrLRS28aF0FE15kQ1kAQIvHCGzGCmw1sT5aenprZJn0YumuCyzAP49N4q5e+jb2bEXQjFjR89VcV1fVnlZIe4/hnL7px89btO1vb/bv37j/YGew+PPW2dgImwirrzjLuQUkDE5So4KxywHWmYJpdHK/q00twXlrzGZcVpJqXRhZScAzSfHDICsdFk7dNji1TUOBsep404bXnDG1FX9KQ7HVJIOZkucCUvqHxfDCMR/E66HVIOhiSLk7mu719lltRazAoFPd+lsQVpg13KIWCts9qDxUXF7yEWUDDNfi0WV/Z0qdByWlhXXgG6Vr9c6Lh2vulzkKn5rjwm7WV+M9apjc2Y3GQNtJUNYIRV4uLWlG0dGUhzaUDgWoZgAsnw9+pWPBgIwaj+332DsJxDj6ERR8rcByte94w7kotTRuOLdmLFf2vkX/53Rgo+JxsunodTvdGyXg0/rQ/PDruHN8ij8kT8owk5DU5Iu/JCZkQQb6Sb+Q7+RFtR6+icXRw1Rr1uplH5K+I3v4CXQ3Grg==</latexit> <latexit sha1_base64="C7WIyqWowAWoqVB4i9nLF/HUqIg=">AAACfHicfZBdaxNBFIYn61eNX6leejMYhfqRsLuU2Buh0F54I1YwbaGbhjOzZ9OhM7PLzFlpWPZv+Gu81f/gn5FO0giaigcGHt7zHs6cV1RaeYrjn53oxs1bt+9s3O3eu//g4aPe5uNDX9ZO4liWunTHAjxqZXFMijQeVw7BCI1H4nxv0T/6gs6r0n6meYUTAzOrCiWBgjTtxRkVDmSTt01O7RacpgNxmr7k73gKWV4SBz7gqViiCGo87fXjYbwsfh2SFfTZqg6mm53tMC1rg5akBu9PkriiSQOOlNTYdrPaYwXyHGZ4EtCCQT9plqe1/EVQcl6ULjxLfKn+OdGA8X5uRHAaoDO/3luI/+wJs7aZip1Jo2xVE1p5tbioNaeSL3LjuXIoSc8DgHQq/J3LMwjRUUi32832MRzn8ENY9LFCB1S6V00GbmaUbcOxs+zNgv5nhIvfxkAh52Q91etwmA6T0XD0abu/u7dKfIM9Zc/YFkvYW7bL3rMDNmaSfWXf2Hf2o/Mreh69jgZX1qizmnnC/qpodAn+h8EE</latexit> む a 6= b <latexit sha1_base64="RKPK5FAsv81nSo+OocCdWGdaIEU=">AAACUHicfZBBSxtBFMffxtrqqm2ix16GhoKIhF2RtMeAHnoRFUwUsiG8nbzE0ZnZZWZWDEu+g1f9WL35TbzZSYxgY+mDYX783394b/5pLoV1UfQYVJY+LH/8tLIarq1vfP5SrW12bFYYTm2eycxcpGhJCk1tJ5yki9wQqlTSeXp9MO2f35CxItNnbpxTT+FIi6Hg6LzUwUQTS/vVetSIZsXeQzyHOszrpF8L9pNBxgtF2nGJ1nbjKHe9Eo0TXNIkTApLOfJrHFHXo0ZFtlfO1p2w714ZsGFm/NGOzdS3L0pU1o5V6p0K3aVd7E3Ff/ZStTDZDX/2SqHzwpHmL4OHhWQuY9Ms2EAY4k6OPSA3wu/O+CUa5M4nFobJIfnPGTryg45zMugys1MmaEZK6In/7CjZndL/jHj7avTkc44XU30Pnb1G3Gw0T/frrYN54ivwFb7BNsTwA1rwC06gDRyu4A7u4SH4HTwFz5Xgxfp6wxb8VZXwD37qtD8=</latexit> 2 b ) = 2aȧ : 2bḃ = 0 <latexit sha1_base64="oP8v2VHPU5MGbyKPFKzgr5IvwlM=">AAACgHicfVBdSxtBFJ2strXpV9THvgyGwqaEuCvBFqEg2Ie+iBYaFdwQ7k5uksH52M7cLYYlf6S/xtf2H/TfdBIjaCy9MHA451zO3JMXSnpKkj+1aG39ydNnG8/rL16+ev2msbl15m3pBPaEVdZd5OBRSYM9kqTwonAIOld4nl8dzfXzH+i8tOYbTQvsaxgbOZICKFCDRjeGdt7in3jmvzuqxCCZxZmwfhJnNEGC3b1WO2jS3CNag0Yz6SSL4Y9BugRNtpzTwWatmw2tKDUaEgq8v0yTgvoVOJJC4ayelR4LEFcwxssADWj0/Wpx3oy/C8yQj6wLzxBfsPc3KtDeT3UenBpo4le1OflPLdcryTT62K+kKUpCI26DR6XiZPm8Oz6UDgWpaQAgnAx/52ICDgSFhuv17DOG4xweh6CTAh2Qde+rDNxYSzMLx46z9hz9zwjXd8aAQs/paquPwdleJ93v7H/tNg+Plo1vsLdsh8UsZR/YIfvCTlmPCfaT3bBf7HcURXG0G6W31qi23NlmDyY6+Av8HsH2</latexit> (a, b) = d✓ ⌧ =s dt p W 2> c0 (cosh(✓/2), sinh(✓/2)) ( i W2 = 0 c0 2 d dt (a d h 1 1> W W dt matrix balancing 2 = a 2 b む <latexit sha1_base64="PW3tAs1QCudVRa9RYJQKp+evktU=">AAACUXicfZBNSwMxEIZn1+/6rUcvi0UQ0bJbRD0KevAiKlgruFVm02kNTbJLkhXL0h/hVX+WJ3+KN9NaQas4EHh45w0z8yaZ4MaG4Zvnj41PTE5Nz5Rm5+YXFpeWV65MmmtGNZaKVF8naEhwRTXLraDrTBPKRFA96Rz1+/UH0oan6tJ2M2pIbCve4gytk+p4W91Jbqt3S+WwEg4q+A3REMowrPO7ZW83bqYsl6QsE2jMTRRmtlGgtpwJ6pXi3FCGrINtunGoUJJpFIN9e8GGU5pBK9XuKRsM1O8/CpTGdGXinBLtvRnt9cU/e4kcmWxbB42Cqyy3pNjn4FYuApsG/TCCJtfErOg6QKa52z1g96iRWRdZqRQfkztO06kbdJaRRpvqrSJG3ZZc9dyx7Xi7T/8Z8fHL6MjlHI2m+huuqpVor7J3sVs+PBomPg1rsA6bEME+HMIJnEMNGHTgCZ7hxXv13n3w/U+r7w3/rMKP8mc/AH6Vs0g=</latexit> 2 − ) <latexit sha1_base64="NvMEsf3MWLINwP/dQEVikUeR0oA=">AAACeHicfVDLShxBFK1p89DJa9RlNkWGkAdm6BYxbgKCWbgRDWRUsIbhdvXtmcKq6qbqdnBo+ifyNW6Tv/BXXFnzEJIx5ELB4ZxzOXVPWmrlKY5vWtHKo8dPnq6utZ89f/HyVWd949QXlZPYl4Uu3HkKHrWy2CdFGs9Lh2BSjWfp5cFUP/uBzqvCfqdJiQMDI6tyJYECNexsCYKKi9yBrDNBYyRo6owa/oV7/onLYSy8suO5Mux04148G/4QJAvQZYs5Ga63dkRWyMqgJanB+4skLmlQgyMlNTZtUXksQV7CCC8CtGDQD+rZWQ1/G5iM54ULzxKfsX9u1GC8n5g0OA3Q2C9rU/KfWmqWkinfG9TKlhWhlfPgvNKcCj7tjGfKoSQ9CQCkU+HvXI4hVEah2XZbfMVwnMOjEHRcogMq3MdagBsZZZtw7EhsTdH/jHB1bwwo9Jwst/oQnG73kt3e7red7v7BovFV9pq9Ye9Zwj6zfXbITlifSfaTXbNf7HfrNuLRu+jD3Bq1Fjub7K+Jtu8AfC/CBw==</latexit> c0 sinh ✓ ; 一 一 行 目 目 一 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 3 3 1 a 📄 b <latexit sha1_base64="067UFn5yTNcR3dGAVXVkIrV15gk=">AAACS3icfZDPSgMxEMaz9V+tf6tHL4tFEJGyK0U9FurBi6hga6FbZDadtqFJdkmyYln6BF71sXwAn8ObeDDbVtAqDgR+fPOFmfnCmDNtPO/Vyc3NLywu5ZcLK6tr6xubxa2GjhJFsU4jHqlmCBo5k1g3zHBsxgpBhBxvw0Et69/eo9IskjdmGGNbQE+yLqNgrHQd3m2WvLI3Lvc3+FMokWld3RWdStCJaCJQGspB65bvxaadgjKMchwVgkRjDHQAPWxZlCBQt9PxpiN3zyodtxsp+6Rxx+r3HykIrYcitE4Bpq9ne5n4Zy8UM5NN97SdMhknBiWdDO4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYhUJwhvY4hRd20GWMCkykDtIAVE8wObLH9oLDjP4zwsOX0ZLN2Z9N9Tc0jsr+cfn4ulKq1qaJ58kO2SX7xCcnpErOyRWpE0qQPJIn8uy8OG/Ou/Mxseac6Z9t8qNyC5+pvLNc</latexit>
  27. む NN {(xi , yi )}ni=1 <latexit sha1_base64="A0WnP2aUlxWqCCxSnZ7ftO30sLE=">AAACYXicfZBNS8NAEIa38avWr7biyUuwCCpSEhH1Igh68CIq2CqYGibbaV3c3YTdjbSE/Biv+os8+0fcfghaxYGFh3feZWbeKOFMG897LzhT0zOzc8X50sLi0vJKuVJt6jhVFBs05rG6i0AjZxIbhhmOd4lCEBHH2+jpdNC/fUalWSxvTD/BloCuZB1GwVgpLK8F2VYvZLv9kG0HeZixYz9/sHrNq3vDcn+DP4YaGddVWCnsB+2YpgKloRy0vve9xLQyUIZRjnkpSDUmQJ+gi/cWJQjUrWy4f+5uWqXtdmJlnzTuUP3+IwOhdV9E1inAPOrJ3kD8sxeJicmmc9TKmExSg5KOBndS7prYHYTjtplCanjfAlDF7O4ufQQF1NgIS6XgDO1xCi/soMsEFZhY7WQBqK5gMrfHdoPdAf1nhN6X0ZLN2Z9M9Tc09+r+Qf3ger92cjpOvEjWyQbZIj45JCfknFyRBqEkIy/klbwVPpx5p+xUR1anMP6zSn6Us/4J0564ng==</latexit> む E(W

    ) = <latexit sha1_base64="rd5oajX48R65mKNFLX36m+8U7g4=">AAACl3icfVDbahRBEO0db3G9bcyT+NK4CCKyzGiIIgQDCRIQY6JuEkjvDj29tZMmfRm6a8IOw3xUvkZ80z+xZ7OCbsSCgsOpU5yqkxVKeozj753o2vUbN2+t3O7euXvv/oPe6sNDb0snYCissu444x6UNDBEiQqOCwdcZwqOsrPtdn50Ds5La75iVcBI89zIqRQcA5X2PrAvMtecvt2kDGGGHisFlPlSp7XcTJqxqVI5S+WYoS0ok4YyzfE0y+rPzbjeS18xlBo83UuTJu3140E8L3oVJAvQJ4vaT1c762xiRanBoFDc+5MkLnBUc4dSKGi6rPRQcHHGczgJ0PBgNarnXzf0aWAmdGpdaIN0zv65UXPtfaWzoGxP9suzlvznLNNLzjh9M6qlKUoEIy6Np6WiaGkbKZ1IBwJVFQAXTobbqTjljgsMwXe7bAfCcw4+BqNPBTiO1j2vGXe5lqYJz+bsRYv+J+Sz38KAQs7JcqpXweHLQbIx2DhY729tLxJfIY/JE/KMJOQ12SK7ZJ8MiSAX5Bv5QX5Gj6J30fto91IadRY7a+Svig5+AeYTzfs=</latexit> ⌃ := む <latexit sha1_base64="rsRj1eWWlrKUfhxnz29b60rvjQc=">AAACY3icfZDRShtBFIYnq7Y2ahttwQspDIZCkRp2RdSbgqAX3kgVjRHcEM5OTjaDM7PLzFkxLHkab/WBfADfw0lMQWPpgYGP//wzZ86f5Eo6CsPHSjAzO/fh4/yn6sLi0ucvteWVC5cVVmBTZCqzlwk4VNJgkyQpvMwtgk4UtpLrg1G/dYPWycyc0yDHtobUyJ4UQF7q1FZjgiLuZsRb/DePz2SqgW/yVqdWDxvhuPh7iCZQZ5M66SxXtv0zotBoSChw7ioKc2qXYEkKhcNqXDjMQVxDilceDWh07XK8wZD/8EqX9zLrjyE+Vl/fKEE7N9CJd2qgvpvujcR/9hI9NZl6e+1SmrwgNOJlcK9QnDI+iod3pUVBauABhJX+71z0wYIgH2K1Gh+iX87isR/0J0cLlNmNMgabammGftk0/jWi/xnh9q/Rk885mk71PVxsNaKdxs7pdn3/YJL4PFtj6+wni9gu22dH7IQ1mWBDdsfu2UPlKVgIVoJvL9agMrnzlb2p4Pszf1q4WA==</latexit> ⌧ Ẇ = ⌃ [Saxe-McClelland-Ganguli ⾒ y = Wx <latexit sha1_base64="CGge8VvQCkNFBMLs8IE7JzFPFiQ=">AAACUHicfZBBSxtBFMffptqmW63aHnsZDIVSJOyK2F4KgfTQSzGCSQR3kbeTt3F0ZnaZmS2GJd+h1/qxvPlNvNVJjNBG8cEwP/7vP7w3/6yUwroougkaL1ZWX75qvg7frK2/3djcejewRWU49XkhC3OcoSUpNPWdcJKOS0OoMknD7KI76w9/kbGi0EduUlKqcKxFLjg6Lw0m7BsbXp5utqJ2NC/2GOIFtGBRvdOtYC8ZFbxSpB2XaO1JHJUurdE4wSVNw6SyVCK/wDGdeNSoyKb1fN0p++iVEcsL4492bK7++6JGZe1EZd6p0J3Z5d5MfLKXqaXJLv+a1kKXlSPN7wfnlWSuYLMs2EgY4k5OPCA3wu/O+Bka5M4nFobJd/KfM/TTDzooyaArzOc6QTNWQk/9Z8fJzoyeM+Llg9GTzzleTvUxDHbb8X57/3Cv1ekuEm/CB9iGTxDDF+jAD+hBHzicw2/4A1fBdXAb/G0E99aHG97Df9UI7wDrArPy</latexit> <latexit sha1_base64="0wRcr1oa3UexjVoLV2D7uLrF3k0=">AAACdnicfZDRShtBFIYnq1WbtjbqpSBDg9RKG3aDqDeCoEJvRAuNEZy4nJ2cTQZnZpeZWTGseQefxtv6Gr6Jl53ECG0sPTDw8Z//cOb8SS6FdWH4WAlmZt/MzS+8rb57/2HxY21p+cxmheHY4pnMzHkCFqXQ2HLCSTzPDYJKJLaTq4NRv32NxopM/3SDHDsKelqkgoPzUlzbPNpof6F7lKUGeNRktlBxKfai4aVmt4NY0G+0fRMLdnvZjGv1sBGOi76GaAJ1MqnTeKmyxboZLxRqxyVYexGFueuUYJzgEodVVljMgV9BDy88alBoO+X4qCFd90qXppnxTzs6Vv+cKEFZO1CJdypwfTvdG4n/7CVqarNLdzul0HnhUPPnxWkhqcvoKDHaFQa5kwMPwI3wf6e8Dz4u53OtVtkh+uMMHvtFJzkacJnZLBmYnhJ66I/tsa8j+p8Rbl6MnnzO0XSqr+Gs2Yi2G9s/tur7B5PEF8gq+UQ2SER2yD75Tk5Ji3ByR+7JL/JQeQrWgvXg87M1qExmVshfFYS/Aarzv70=</latexit> W Pn > N3 ⇥N1 y x 2 R ; i=1 i i n X 1 2 i=1 2 kyi W xi k Pn <latexit sha1_base64="xjOxvmjR64ptkxcnbTP+RZUHXVA=">AAACfXicfZDbahsxEIblTQ+Je3LSy96ImkApweyGkPQmEEgu2ou2KdRJIHKWWXnsiOiwlWaDzbLP0afpbfsMfZpUdlxondJBgo9/fjGavyi1CpSmP1vJyr37Dx6urrUfPX7y9FlnfeMkuMpL7EunnT8rIKBWFvukSONZ6RFMofG0uDqc9U+v0Qfl7GealjgwMLZqpCRQlPJOJggnFGiqkYtQmbxW+1lzYSe5iudCkCu5wC+Vuubv8vpDnjV5p5v20nnxu5AtoMsWdZyvt3bE0MnKoCWpIYTzLC1pUIMnJTU2bVEFLEFewRjPI1owGAb1fLeGb0ZlyEfOx2uJz9U/X9RgQpiaIjoN0GVY7s3Ef/YKszSZRm8GtbJlRWjl7eBRpTk5PguOD5VHSXoaAaRX8e9cXoIHSTHedlscYVzO4/s46GOJHsj517UAPzbKNnHZsdia0f+MMPltjBRzzpZTvQsn271st7f7aad7cLhIfJW9YC/ZK5axPXbA3rJj1meSfWXf2Hf2o3WTbCZbSe/WmrQWb56zvyrZ+wXtY8T8</latexit> ⌃ = U SV > > x x i=1 i i ⌘ IN1 > <latexit sha1_base64="NXupUhNl5crBqocmcqCFCVvbpy4=">AAACYHicfVBdSxtBFJ2stU1jWxP1qX0ZGgRbStgtor4Igj74UmqhMYKbhruTmzg4H8vMXTEu+S99rf/I1/4SZ2MKNZYeGDhz7rnce0+WK+kpju9q0dKz5ecv6i8bK69ev1ltttZOvS2cwK6wyrqzDDwqabBLkhSe5Q5BZwp72eVhVe9dofPSmu80ybGvYWzkSAqgIA2aG7BFH/g+L36kZHPeq35Xg2Y77sQz8KckmZM2m+Nk0Kptp0MrCo2GhALvz5M4p34JjqRQOG2khcccxCWM8TxQAxp9v5ytP+WbQRnykXXhGeIz9e+OErT3E50Fpwa68Iu1SvxnLdMLk2m01y+lyQtCIx4GjwrFyfIqGz6UDgWpSSAgnAy7c3EBDgSFBBuN9AjDcQ6/hEFfc3RA1n0sU3BjLc00HDtOP1Xsf0a4/mMMLOScLKb6lJx+7iQ7nZ1v2+2Dw3nidfaOvWdbLGG77IAdsxPWZYLdsJ/sF7ut/Y7q0WrUerBGtXnPOnuE6O09Yg224w==</latexit> <latexit sha1_base64="gqsoMc39D4bwJRNI9IXCMxyR9zQ=">AAACXXicfZDfShtBFMYna602tTaxF73wZjAUpJSwW0S9EQR74U2pYhMFN4azk5N1cP4sM2fVsORRvNVn8qqv0klMoY3FAwM/vvMN55wvK5T0FMePtWjh1eLrpeU39bcr71bfN5prXW9LJ7AjrLLuLAOPShrskCSFZ4VD0JnC0+zqYNI/vUbnpTU/aVRgT0Nu5FAKoCD1G830ROYa+B7vnHQvUrJFv9GK2/G0+HNIZtBiszrqN2tb6cCKUqMhocD78yQuqFeBIykUjutp6bEAcQU5ngc0oNH3qunuY/4pKAM+tC48Q3yq/v2jAu39SGfBqYEu/XxvIv63l+m5yTTc7VXSFCWhEU+Dh6XiZPkkGD6QDgWpUQAQTobdubgEB4JCfPV6+g3DcQ6/h0E/CnRA1n2uUnC5lmYcjs3TLxN6yQi3f4yBQs7JfKrPofu1nWy3t4+3WvsHs8SX2TrbYJssYTtsnx2yI9Zhgt2wO3bPHmq/osVoJVp9ska12Z8P7J+KPv4Gphi2ow==</latexit> a(t) = u W (t)v <latexit sha1_base64="aU52D6XpcGvichH7RskOtBpIihM=">AAACZHicfZDRShtBFIYna61pqjUaBKEgQ4OgRcOuiHpTEPSiN0ULRgU3hLOTk2RwZnaZOSuGJT6Nt/V9+gI+h5MYQWPpgYGP//wzZ86fZEo6CsO/pWDmw+zHufKnyuf5hS+L1aXlc5fmVmBTpCq1lwk4VNJgkyQpvMwsgk4UXiTXR6P+xQ1aJ1NzRoMMWxp6RnalAPJSu7oaE+RxJyUOG7TJf3DHt8fYrtbDRjgu/h6iCdTZpE7bS6Vd/47INRoSCpy7isKMWgVYkkLhsBLnDjMQ19DDK48GNLpWMV5hyNe90uHd1PpjiI/V1zcK0M4NdOKdGqjvpnsj8Z+9RE9Npu5Bq5AmywmNeB7czRWnlI/y4R1pUZAaeABhpf87F32wIMinWKnEx+iXs/jLDzrJ0AKl9nsRg+1paYZ+2V68NaL/GeH2xejJ5xxNp/oeznca0V5j7/du/fBokniZfWXf2AaL2D47ZD/ZKWsywe7YPfvDHkqPwXxQC1aerUFpcqfG3lSw9gQ25rgm</latexit> ⌧ ȧ(t) = s a(t) ODE <latexit sha1_base64="yeBAXjFLPW/2a0rC3dgKmWFSri8=">AAACbHicfZDRShtBFIYnW1s12hq1d1IYDJakatwVsb0pCPbCG1HBqOCm4ezkJA7OzC4zZ4thyW2fxtv6Lr6Ez+AkRrCx9MDAx3/+w5nzJ5mSjsLwvhS8mXr7bnpmtjw3//7DQmVx6cyluRXYFKlK7UUCDpU02CRJCi8yi6AThefJ9f6wf/4LrZOpOaV+hi0NPSO7UgB5qV3hUKM6/84dX+c1qIV1vsldHX8Wm7QVE+SDdqUaNsJR8dcQjaHKxnXcXiztxJ1U5BoNCQXOXUZhRq0CLEmhcFCOc4cZiGvo4aVHAxpdqxidMuBrXunwbmr9M8RH6suJArRzfZ14pwa6cpO9ofjPXqInNlP3W6uQJssJjXha3M0Vp5QPc+IdaVGQ6nsAYaX/OxdXYEGQT7Ncjn+gP87ioV90lKEFSu2XIgbb09IM/LG9eGNI/zPCzbPRk885mkz1NZxtN6Ldxu7JTnVvf5z4DFthq6zGIvaV7bEDdsyaTLDf7Jb9YXelh+BjsBJ8erIGpfHMMvurgs+Pumi6KQ==</latexit> a(t) = s + (a(0) s)e t/⌧ 小 目 行 比 力 入 二 4 1 行 力 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 4 入 3 一 📄 2 ]
  28. NN [Saxe-McClelland-Ganguli 2 NN Ricatti ODE <latexit sha1_base64="aU52D6XpcGvichH7RskOtBpIihM=">AAACZHicfZDRShtBFIYna61pqjUaBKEgQ4OgRcOuiHpTEPSiN0ULRgU3hLOTk2RwZnaZOSuGJT6Nt/V9+gI+h5MYQWPpgYGP//wzZ86fZEo6CsO/pWDmw+zHufKnyuf5hS+L1aXlc5fmVmBTpCq1lwk4VNJgkyQpvMwsgk4UXiTXR6P+xQ1aJ1NzRoMMWxp6RnalAPJSu7oaE+RxJyUOG7TJf3DHt8fYrtbDRjgu/h6iCdTZpE7bS6Vd/47INRoSCpy7isKMWgVYkkLhsBLnDjMQ19DDK48GNLpWMV5hyNe90uHd1PpjiI/V1zcK0M4NdOKdGqjvpnsj8Z+9RE9Npu5Bq5AmywmNeB7czRWnlI/y4R1pUZAaeABhpf87F32wIMinWKnEx+iXs/jLDzrJ0AKl9nsRg+1paYZ+2V68NaL/GeH2xejJ5xxNp/oeznca0V5j7/du/fBokniZfWXf2AaL2D47ZD/ZKWsywe7YPfvDHkqPwXxQC1aerUFpcqfG3lSw9gQ25rgm</latexit> ⌧ ȧ(t)

    = s <latexit sha1_base64="AaPjfFRssFLAVnlEV/g/ZEAVNyA=">AAACYHicfVDRShtBFJ1sbY1p1aT2SV+GBkFFw66I7UtB0AdfRAuNCm4Idyc3cXBmdpm5U5ou+Rdf2z/qq1/iJEbQKL0wcDj33Dn3nqxQ0lEc/6tEb+bevpuvLtTef1hcWq43Pp673FuBbZGr3F5m4FBJg22SpPCysAg6U3iR3RyO+xc/0TqZmx80LLCjYWBkXwqgQHXrn1ICn/Zy4p5/47t+w+34zW69GbfiSfGXIJmCJpvWWbdR2Qt/CK/RkFDg3FUSF9QpwZIUCke11DssQNzAAK8CNKDRdcrJ+iO+Hpge7+c2PEN8wj6dKEE7N9RZUGqgazfbG5Ov9jI940z9r51SmsITGvFg3PeKU87H2fCetChIDQMAYWXYnYtrsCAoJFirpUcYjrN4EoxOC7RAud0qU7ADLc0oHDtIt8fof0L49SgMKOSczKb6EpzvtpL91v73vebB4TTxKltjn9kGS9gXdsCO2RlrM8F+s1v2h/2t3EXVaDlqPEijynRmhT2raPUeRB63VA==</latexit> a(t) ⌧ u̇ = 2u(s u) <latexit sha1_base64="JVHaV/mWi3LZKAaydpHCnmFQoXk=">AAAChnicfZDbahsxEIbl7SGue4jTXPZG1BScHpzdkBOFgiG96E1oCnESyLpmVp51RCTtIs2WGrHv0qfpbXvbt6nsOJA4pQOCT//8YjR/VirpKI7/NKJ79x88XGk+aj1+8vTZanvt+YkrKitwIApV2LMMHCppcECSFJ6VFkFnCk+zy4NZ//QbWicLc0zTEocaJkbmUgAFadR+X3Vpg3/gaW5BeIdf/ZajzZSgqmt/88bf8YS/4d7Vm77qxht1PWp34l48L34XkgV02KKORmuN7XRciEqjIaHAufMkLmnowZIUCutWWjksQVzCBM8DGtDohn6+ZM1fBWXM88KGY4jP1ZsvPGjnpjoLTg104ZZ7M/GfvUwvTaZ8f+ilKStCI64G55XiVPBZgnwsLQpS0wAgrAx/5+ICQnwUcm610o8YlrN4GAZ9LtECFfa1T8FOtDR1WHaSvp3R/4zw/doYKOScLKd6F062eslub/fLdqd/sEi8yV6wl6zLErbH+uwTO2IDJtgP9pP9Yr+jZtSLdqK9K2vUWLxZZ7cq6v8FfTfFXg==</latexit> se2st/⌧ u(t) = 2st/⌧ e 1 + s/u(0) <latexit sha1_base64="yeBAXjFLPW/2a0rC3dgKmWFSri8=">AAACbHicfZDRShtBFIYnW1s12hq1d1IYDJakatwVsb0pCPbCG1HBqOCm4ezkJA7OzC4zZ4thyW2fxtv6Lr6Ez+AkRrCx9MDAx3/+w5nzJ5mSjsLwvhS8mXr7bnpmtjw3//7DQmVx6cyluRXYFKlK7UUCDpU02CRJCi8yi6AThefJ9f6wf/4LrZOpOaV+hi0NPSO7UgB5qV3hUKM6/84dX+c1qIV1vsldHX8Wm7QVE+SDdqUaNsJR8dcQjaHKxnXcXiztxJ1U5BoNCQXOXUZhRq0CLEmhcFCOc4cZiGvo4aVHAxpdqxidMuBrXunwbmr9M8RH6suJArRzfZ14pwa6cpO9ofjPXqInNlP3W6uQJssJjXha3M0Vp5QPc+IdaVGQ6nsAYaX/OxdXYEGQT7Ncjn+gP87ioV90lKEFSu2XIgbb09IM/LG9eGNI/zPCzbPRk885mkz1NZxtN6Ldxu7JTnVvf5z4DFthq6zGIvaV7bEDdsyaTLDf7Jb9YXelh+BjsBJ8erIGpfHMMvurgs+Pumi6KQ==</latexit> a(t) = s + (a(0) : 2 s)e t/⌧ NN 比 4 1 非 Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. 5 3 📄 2 ]
  29. 1 min (yi f 2F n i=1 己 己 yi

    7 自 3 自 n X <latexit sha1_base64="zd0qs9YnZx3rv+u8W9SoszuY7LY=">AAACh3icfVBdSxtBFJ2s/dC01WgffRkahCg27oqkghQUS+lLUaFRwYnL3cndODgzu8zMimHZH9Nf09f20X/jbEyhRumFgcO553LmnCSXwrowvGsEcy9evno9v9B88/bd4lJreeXUZoXh2OeZzMx5Ahal0Nh3wkk8zw2CSiSeJdeH9f7sBo0Vmf7hxjkOFIy0SAUH56m4tceU0HGZMqGZAnfFQdKvFWWpAR5pZgsVl+JzVF1q2hnHgn6kaec2Fuvr9HI7brXDbjgZ+hREU9Am0zmOlxs7bJjxQqF2XIK1F1GYu0EJxgkusWqywmIO/BpGeOGhBoV2UE5SVnTNM0OaZsY/7eiE/feiBGXtWCVeWSexs7uafHaXqBlnl+4OSqHzwqHmD8ZpIanLaF0hHQqD3MmxB8CN8H+n/Ap8Yc4X3WyyL+jDGfzujY5yNOAys1EyMCNfdeXDjthmjf4nhNu/Qo98z9Fsq0/B6XY36nV7Jzvt/cNp4/NklXwgHRKRT2SffCPHpE84+Ul+kd/kT7AQbAW9YPdBGjSmN+/JowkO7gGTOMXH</latexit> n X <latexit sha1_base64="k8IjcVVg0ced+68bAA+pKfkHlGg=">AAACfXicfZDbahRBEIZ7x1NcTxu99KZxCUQJy4yExBshEC+8ESO4SSC9DjW9NbNN+jB018guwz6HT+OtPoNPo72bEXQjFjR8/PUX1fUXtVaB0vRHL7lx89btO1t3+/fuP3j4aLD9+DS4xkscS6edPy8goFYWx6RI43ntEUyh8ay4PF71zz6jD8rZj7SocWKgsqpUEihK+SATBua5oBkScFF6kJkVoTF5q15ny09WaFfxujPsznP1PB8M01G6Ln4dsg6GrKuTfLu3L6ZONgYtSQ0hXGRpTZMWPCmpcdkXTcAa5CVUeBHRgsEwade3LflOVKa8dD4+S3yt/jnRgglhYYroNECzsNlbif/sFWZjM5WvJq2ydUNo5dXistGcHF8Fx6fKoyS9iADSq/h3LmcQA6MYb78v3mA8zuO7uOh9jR7I+RetAF8ZZZfx2Ersreh/Rpj/NkaKOWebqV6H05ej7GB08GF/eHTcJb7FnrJnbJdl7JAdsbfshI2ZZF/YV/aNfe/9THaSvWR0ZU163cwT9lclh78A1prEcw==</latexit> f (xi )) 2 xi <latexit sha1_base64="h8lWl2VcAk8a9cUwBnalCUZxLC4=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiUXRH1KNSDF1HR2kK3lNl0uoYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88vLBaWlldW14ql9Vsdp4pijcY8Vo0QNHImsWaY4dhIFIIIOdbDXnXYr9+j0iyWN6afYEtAJFmXUTBWun5os3ax7FW8UbnT4I+hTMZ12S45B0EnpqlAaSgHrZu+l5hWBsowynFQCFKNCdAeRNi0KEGgbmWjXQfutlU6bjdW9knjjtSfPzIQWvdFaJ0CzJ2e7A3FP3uhmJhsusetjMkkNSjp1+Buyl0Tu8Mg3A5TSA3vWwCqmN3dpXeggBobV6EQnKI9TuG5HXSRoAITq90sABUJJgf22CjYG9J/Rnj4NlqyOfuTqU7D7X7FP6wcXh2UT6rjxBfIJtkiO8QnR+SEnJFLUiOUROSRPJFn58V5c96djy9rzhn/2SC/Kpf/BJwLtE4=</latexit> ⾒ si <latexit sha1_base64="roEqQMiNvGE//dX6FbSz1oYgNug=">AAACTXicfZBNSwMxEIaz9aO1flU9elksgoiUXRH1KNSDF1HRWqFbymw63YYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88XFoqLS8srq6W19Vsdp4pijcY8VnchaORMYs0ww/EuUQgi5FgPe9Vhv36PSrNY3ph+gk0BkWQdRsFY6Vq3WKtU9ireqNxp8MdQJuO6bK05B0E7pqlAaSgHrRu+l5hmBsowynFQDFKNCdAeRNiwKEGgbmajXQfutlXabidW9knjjtSfPzIQWvdFaJ0CTFdP9obin71QTEw2neNmxmSSGpT0a3An5a6J3WEQbpsppIb3LQBVzO7u0i4ooMbGVSwGp2iPU3huB10kqMDEajcLQEWCyYE9Ngr2hvSfER6+jZZszv5kqtNwu1/xDyuHVwflk+o48QLZJFtkh/jkiJyQM3JJaoSSiDySJ/LsvDhvzrvz8WXNOeM/G+RX5fKfkn60SQ==</latexit> yi <latexit sha1_base64="BbkNUAiU7AyoEFOqIMluMR6TRus=">AAACTXicfZBNSwMxEIaz9aO1flU9elksgoiUXRH1KNSDF1HRWqFbymw63YYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88XFoqLS8srq6W19Vsdp4pijcY8VnchaORMYs0ww/EuUQgi5FgPe9Vhv36PSrNY3ph+gk0BkWQdRsFY6brfYq1S2at4o3KnwR9DmYzrsrXmHATtmKYCpaEctG74XmKaGSjDKMdBMUg1JkB7EGHDogSBupmNdh2421Zpu51Y2SeNO1J//shAaN0XoXUKMF092RuKf/ZCMTHZdI6bGZNJalDSr8GdlLsmdodBuG2mkBretwBUMbu7S7uggBobV7EYnKI9TuG5HXSRoAITq90sABUJJgf22CjYG9J/Rnj4NlqyOfuTqU7D7X7FP6wcXh2UT6rjxAtkk2yRHeKTI3JCzsglqRFKIvJInsiz8+K8Oe/Ox5c154z/bJBflct/Ap30tE8=</latexit> 1 max log p✓ (xi ) ✓ n i=1 yi
  30. 2020: (data augmentation) 己 自 己 Chen, T., Kornblith, S.,

    Norouzi, M., & Hinton, G. (2020). A simple framework for contrastive learning of visual representations. 8 自 3 📄 1993:
  31. Radford et al. (2021). Learning transferable visual models from natural

    language supervision. 9 3 📄 CLIP (Contrastive Language-Image Pretraining)
  32. SimCLR [Chen+ xi <latexit sha1_base64="OYIuD3tQeI2w0dbax8wVkezfngY=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7Xou3Fspepeb5tZ1dOoC97SFUa9SveP0qk2GdtRadtaCdsCxGaZgArRu+l5pmDspwJrDnBpnGFNgtdLBhUUKMupn3h+7RNau0aZQoe6ShffXrjRxirbtxaJ3FkPpnrxB/6zUyE+01cy7TzKBkg4+iTFCT0CIB2uYKmRFdC8AUt7NSdgMKmLE5uW5whHYZhSf24dMUFZhEbeQBqE7MZc8u1wk2C/rPCPefRkuuDfYzPfo3XG5V/Gqler5d3j8YRjxJVsgqWSc+2SX75JickTphJCUP5JE8Oc/Oq/PmvA+sJWd4Z5l8q5LzAQY1sjc=</latexit> xj <latexit sha1_base64="oBHF1h2GFlEaPhB/HhOweL1oh7I=">AAACR3icfZDfShtBFMZnY/3TrdWol70ZGoQiJexKTMxd0F54U4zQxEA2hLOTs8mY2dllZlYMS16jt+3r9BH6FN6VXnY2iVBFPTDw4zvfzJzzhang2njeb6e09mZ9Y3Prrftu+/3Obnlvv6uTTDHssEQkqheCRsEldgw3AnupQohDgdfh9LzoX9+i0jyR38wsxUEMY8kjzsBYKQhiMJMwyu/mw5thueJVm57fPGnQJZzWVlBvUr/qLapCVtUe7jmHwShhWYzSMAFa930vNYMclOFM4NwNMo0psCmMsW9RQox6kC+GntNDq4xolCh7pKEL9f8bOcRaz+LQOosh9dNeIT7X62cmOh3kXKaZQcmWH0WZoCahRQJ0xBUyI2YWgCluZ6VsAgqYsTm5bvAF7TIKv9qHL1NUYBJ1lAegxjGXc7vcOPhc0GtGuHswWnJtsA/p0Zehe1z169X6Va3SOltFvEU+kI/kE/FJg7TIBWmTDmEkJd/JD/LT+eXcO3+cv0tryVndOSCPquT8AwgNsjg=</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit>

    <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> x j x j x j x xjj ] zi Encoder <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> zj <latexit sha1_base64="g841n+aIva7pMukoNWeQg2B5H4A=">AAACR3icfZDLbhMxFIY9oUCYcklgycZqFKmqUDRThVx2EbBgUzWVmjRSJorOOGcSE49nZHuqhlFegy28Do/AU7CrWOLJRSoVcCRLn/7z2z7nD1PBtfG8H07pwcHDR4/LT9zDp8+ev6hUXw51kimGA5aIRI1C0Ci4xIHhRuAoVQhxKPAqXL4v+lfXqDRP5KVZpTiJYS55xBkYKwVBDGYRRvnn9fTTtFLzGl3P775t0y10mjtodanf8DZVI7vqT6tOPZglLItRGiZA67HvpWaSgzKcCVy7QaYxBbaEOY4tSohRT/LN0Gtat8qMRomyRxq6Ue/eyCHWehWH1lkMqe/3CvFvvXFmos4k5zLNDEq2/SjKBDUJLRKgM66QGbGyAExxOytlC1DAjM3JdYMPaJdReGYfPk9RgUnUSR6Amsdcru1y8+BNQf8zws3eaMm1we7To/+G4WnDbzVaF81a790u4jJ5TY7IMfFJm/TIR9InA8JISr6Qr+Sb89356dw6v7bWkrO784r8USXnNwvDsjo=</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> z j z j z j z zjj 0 2 Chen, T., Kornblith, S., Norouzi, M., & Hinton, G. (2020). A simple framework for contrastive learning of visual representations. 0 4 📄 (contrastive learning)
  33. SimSiam [Chen-He ] xi hi pred head hj Stop Grad

    <latexit sha1_base64="M7L9l5V9YJ8k17bUYdQox0J08kU=">AAACR3icfZDPSiNBEMZ7ou5mR3c1etxLYxBEljAjmj+3sHrYixjBqJAJoaZTkzT29AzdPYthyGt4XV/HR/ApvMke7Uki6LJa0PDjq6+7q74wFVwbz3twSkvLK58+l7+4q2tfv61vVDYvdJIphl2WiERdhaBRcIldw43Aq1QhxKHAy/D6qOhf/kaleSLPzSTFfgwjySPOwFgpCGIw4zDKx9MBH2xUvVrL81uHDTqH5sEC6i3q17xZVcmiOoOKsxMME5bFKA0ToHXP91LTz0EZzgRO3SDTmAK7hhH2LEqIUffz2dBTumOVIY0SZY80dKa+vpFDrPUkDq2zGFL/2yvE//V6mYma/ZzLNDMo2fyjKBPUJLRIgA65QmbExAIwxe2slI1BATM2J9cNjtEuo/DEPnyaogKTqL08ADWKuZza5UbBj4I+MsLNi9GSa4N9SY++Dxf7Nb9eq58dVNs/FxGXyXeyTXaJTxqkTX6RDukSRlJyS/6QO+feeXSenL9za8lZ3Nkib6rkPAPodrIn</latexit> Encoder <latexit sha1_base64="OYIuD3tQeI2w0dbax8wVkezfngY=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7Xou3Fspepeb5tZ1dOoC97SFUa9SveP0qk2GdtRadtaCdsCxGaZgArRu+l5pmDspwJrDnBpnGFNgtdLBhUUKMupn3h+7RNau0aZQoe6ShffXrjRxirbtxaJ3FkPpnrxB/6zUyE+01cy7TzKBkg4+iTFCT0CIB2uYKmRFdC8AUt7NSdgMKmLE5uW5whHYZhSf24dMUFZhEbeQBqE7MZc8u1wk2C/rPCPefRkuuDfYzPfo3XG5V/Gqler5d3j8YRjxJVsgqWSc+2SX75JickTphJCUP5JE8Oc/Oq/PmvA+sJWd4Z5l8q5LzAQY1sjc=</latexit> xj <latexit sha1_base64="gwzGj/YlW7S86DEYY7wqqSaKG3g=">AAACR3icfZDNSiNBFIWrM/62P2N06aYwCCISusWJyU7GWbgRFYwK6RBuV24npdXVTVW1GJq8hlvndeYR5inciUurkwgq6oWCj3NPVd17wlRwbTzvv1P6MTU9Mzs37y4sLi3/XCmvXugkUwybLBGJugpBo+ASm4YbgVepQohDgZfhzWHRv7xFpXkiz80gxXYMPckjzsBYKQhiMP0wyvvDznVnpeJVG57f+LVPx1Dfm0CtQf2qN6oKmdRpp+xsBt2EZTFKwwRo3fK91LRzUIYzgUM3yDSmwG6ghy2LEmLU7Xw09JBuWqVLo0TZIw0dqW9v5BBrPYhD6yyG1B97hfhZr5WZqN7OuUwzg5KNP4oyQU1CiwRolytkRgwsAFPczkpZHxQwY3Ny3eAP2mUUHtuHT1JUYBK1nQegejGXQ7tcL9gp6Dsj3L0aLbk22Nf06NdwsVv1a9Xa2V7l4Pck4jmyTjbIFvHJPjkgR+SUNAkjKbknD+Sv8895dJ6c57G15EzurJF3VXJeAOpOsig=</latexit> <latexit sha1_base64="oBHF1h2GFlEaPhB/HhOweL1oh7I=">AAACR3icfZDfShtBFMZnY/3TrdWol70ZGoQiJexKTMxd0F54U4zQxEA2hLOTs8mY2dllZlYMS16jt+3r9BH6FN6VXnY2iVBFPTDw4zvfzJzzhang2njeb6e09mZ9Y3Prrftu+/3Obnlvv6uTTDHssEQkqheCRsEldgw3AnupQohDgdfh9LzoX9+i0jyR38wsxUEMY8kjzsBYKQhiMJMwyu/mw5thueJVm57fPGnQJZzWVlBvUr/qLapCVtUe7jmHwShhWYzSMAFa930vNYMclOFM4NwNMo0psCmMsW9RQox6kC+GntNDq4xolCh7pKEL9f8bOcRaz+LQOosh9dNeIT7X62cmOh3kXKaZQcmWH0WZoCahRQJ0xBUyI2YWgCluZ6VsAgqYsTm5bvAF7TIKv9qHL1NUYBJ1lAegxjGXc7vcOPhc0GtGuHswWnJtsA/p0Zehe1z169X6Va3SOltFvEU+kI/kE/FJg7TIBWmTDmEkJd/JD/LT+eXcO3+cv0tryVndOSCPquT8AwgNsjg=</latexit> Ex Ex+ |x <latexit sha1_base64="B60OXPNVQZgwkhoQdFf8fovTB+s=">AAADTXicfVJbb9MwFHYyYCNcduGRF4sKaWVdlaBpTHuaNJB4QYxBt0l1WjnuSWLNuch2UKuQn8Ar/Cye+SG8IYTTG11XcSRLn77zHX/HxyfIBVfadX9a9tqdu/fWN+47Dx4+ery5tb1zobJCMuiwTGTyKqAKBE+ho7kWcJVLoEkg4DK4Pq3zl59BKp6ln/QoBz+hUcpDzqg2VH/bQiThab8kqgiUpuy6JAnVcRDi8BhPYFCeV70BJjqbEfi8V8YVJgTPxNHxQi6+of1otPteVVXz695U/bnL0NCt1ZneHv6CF3UCQt119omgaSQA/zPfnfc8R8Nms4VX0L29JpHjeodIHsXa72813LY7DnwbeFPQQNM4MzM7IIOMFQmkmgmqVNdzc+2XVGrOBFQOKRTkZpQ0gq6BKU1A+eX4ryr83DADHGbSnFTjMbtYUdJEqVESGGXds1rO1eTKXJAsOevwyC95mhcaUjYxDguBzdfUi4AHXALTYmQAZZKb3jGLqaRMm3VxHPIazOMkvDNG73OQVGfyRUmojMy+VOaxEWnV6H9COpwJDTJz9panehtcvGx7h+3DDweNk9PpxDfQU/QM7SIPvUIn6C06Qx3ErMj6an2zvts/7F/2b/vPRGpb05on6Easrf8FvHoOlw==</latexit> min d f :R !R h h g:R !S h 1 ⇥ 1 2 zi <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> + hg(f (x)), f (x )i Chen, X. & He, K. (2021). Exploring simple siamese representation learning. 4 4 非 📄 (Non-contrastive learning) ⇤
  34. SimSiam [Chen-He ] <latexit sha1_base64="B60OXPNVQZgwkhoQdFf8fovTB+s=">AAADTXicfVJbb9MwFHYyYCNcduGRF4sKaWVdlaBpTHuaNJB4QYxBt0l1WjnuSWLNuch2UKuQn8Ar/Cye+SG8IYTTG11XcSRLn77zHX/HxyfIBVfadX9a9tqdu/fWN+47Dx4+ery5tb1zobJCMuiwTGTyKqAKBE+ho7kWcJVLoEkg4DK4Pq3zl59BKp6ln/QoBz+hUcpDzqg2VH/bQiThab8kqgiUpuy6JAnVcRDi8BhPYFCeV70BJjqbEfi8V8YVJgTPxNHxQi6+of1otPteVVXz695U/bnL0NCt1ZneHv6CF3UCQt119omgaSQA/zPfnfc8R8Nms4VX0L29JpHjeodIHsXa72813LY7DnwbeFPQQNM4MzM7IIOMFQmkmgmqVNdzc+2XVGrOBFQOKRTkZpQ0gq6BKU1A+eX4ryr83DADHGbSnFTjMbtYUdJEqVESGGXds1rO1eTKXJAsOevwyC95mhcaUjYxDguBzdfUi4AHXALTYmQAZZKb3jGLqaRMm3VxHPIazOMkvDNG73OQVGfyRUmojMy+VOaxEWnV6H9COpwJDTJz9panehtcvGx7h+3DDweNk9PpxDfQU/QM7SIPvUIn6C06Qx3ErMj6an2zvts/7F/2b/vPRGpb05on6Easrf8FvHoOlw==</latexit> min f :Rd !Rh g:Rh

    !Sh 1 Ex Ex+ |x ⇥ + hg(f (x)), f (x )i む 1 む (f ⌘ c) <latexit sha1_base64="IfmxcWwzDFHiaOkjbXtOHUHxdb8=">AAACXnicfZBNSwMxEIbT9bt+Vb0IXoJFEJGyK6IeC/XgRaxgVXBLmU1nazDJrklWLEv/ilf9S978KWZrBW3FgcDDO2+YmTdKBTfW999L3tT0zOzc/EJ5cWl5ZbWytn5tkkwzbLFEJPo2AoOCK2xZbgXephpBRgJvoodG0b95Qm14oq5sP8W2hJ7iMWdgndSprIcS7H0U05iG+JjxJ8o6lapf84dFJyEYQZWMqtlZKx2G3YRlEpVlAoy5C/zUtnPQljOBg3KYGUyBPUAP7xwqkGja+XD5Ad1xSpfGiXZPWTpUf/7IQRrTl5FzFqua8V4h/tmL5NhkG5+0c67SzKJiX4PjTFCb0CIZ2uUamRV9B8A0d7tTdg8amHX5lcvhKbrjNJ67QRcparCJ3stD0D3J1cAd2wv3C/rPCM/fRkcu52A81Um4PqgFR7Wjy8NqvTFKfJ5skW2ySwJyTOrkjDRJizDyTF7IK3krfXiz3rK3+mX1SqM/G+RXeZufFnm3VQ==</latexit> む ( ) rF (x, g(x)) = @x F + @y F · rx g <latexit sha1_base64="BPvVFsY86WW3ATAP2ihs5Vg1D8o=">AAACjHicfVBdaxNBFJ1s/ahRa1offRkMQqol7EqoQikEKsEXsYJpC90Q7k5utkPnY5m5KwlL/k5/ja8K/htnk4iaihcGzj3nXs7ckxVKeorjH41o687de/e3HzQfPnq886S1u3fmbekEDoVV1l1k4FFJg0OSpPCicAg6U3ieXZ/U+vkXdF5a85nmBY405EZOpQAK1LjVTw1kCvigMzvgeWe2v8+PeVqAIwlqPOMD/up3Ow9tKiaW+Gor6Pm41Y678bL4bZCsQZut63S82+ilEytKjYaEAu8vk7igUVV7CIWLZlp6LEBcQ46XARrQ6EfV8tQFfxGYCZ9aF54hvmT/3KhAez/XWZjUQFd+U6vJf2qZ3nCm6dtRJU1REhqxMp6WipPldY58Ih0KUvMAQDgZ/s7FFTgQFNJuNtN3GI5z+CEYfSzQAVn3skrB5VqaRTg2Tw9q9L9BmP0aDCjknGymehucve4mh93DT712/2Sd+DZ7xp6zDkvYG9Zn79kpGzLBbthX9o19j3aiXnQUHa9Go8Z65yn7q6LBT4kSxp0=</latexit> (StopGrad ) rF (x, StopGrad(g(x))) = @x F + @y F · 0 = @x F <latexit sha1_base64="+Tz4Jf5yOCpxJCcGcOfD2CsxWVI=">AAACpXicfVHbbhMxEHWWWwmXpvDIi0WESKCKdlFVeKlUqajwUlFEkxbV0WrW62yt+rKyZ6tEq3wb38EH8Ep/od40CEgRI1k6c+aMx3OclUp6jOPvrejW7Tt3763dbz94+OjxemfjycjbynEx5FZZd5KBF0oaMUSJSpyUToDOlDjOzvea+vGFcF5ac4SzUow1FEZOJAcMVNr5ygxkCuh+b7pJmQY8c7r+grb84CCf94retN/v0x3KSnAoQaVTuk9f/05nIWU8t0jjFVXa6caDeBH0JkiWoEuWcZhutLZYbnmlhUGuwPvTJC5xXDdXciXmbVZ5UQI/h0KcBmhACz+uFx7M6YvA5HRiXTgG6YL9s6MG7f1MZ0HZbOlXaw35z1qmVybj5N24lqasUBh+PXhSKYqWNgbTXDrBUc0CAO5keDvlZ+CAY/iGdpu9F2E5Jw7CoE+lcIDWvaoZuEJLMw/LFmyzQf8TwvSXMKDgc7Lq6k0wejNItgfbn7e6u3tLx9fIM/Kc9EhC3pJd8pEckiHh5Bv5QX6Sy+hldBAdRaNradRa9jwlf0WUXgGs1NBk</latexit> 自 入 非 1 2 用 力 入 力 Chen, X. & He, K. (2021). Exploring simple siamese representation learning. 5 4 非 📄 (Non-contrastive learning) ⇤
  35. SimSiam [Chen-He min d f :R !R h h g:R

    !S ] Ex Ex+ |x <latexit sha1_base64="9WSF9hO9iCGzJ0i5pz1CGM6hyoc=">AAADYHicfVJdb9MwFHUaYCV8bGVv8GJRIa1sqxI0DbSnSQPBC2JsdJtUt5XjOIk150O2g1pM/guv8I945ZfgtGnpuoorWTo69/ie6+vr55xJ5bq/rYZ95+69jeZ958HDR483t1pPLmRWCEJ7JOOZuPKxpJyltKeY4vQqFxQnPqeX/vVJlb/8SoVkWfpFTXI6SHCUspARrAw1alnbKGHpSCNZ+FJhcq1RglXshzA8gjPo67NyGECksjkBz4Y6LiFCcC6OjpZy8Q3tudHue2VZLsq9K0cLl7Gh99ZnhrvwO1zWcRqqvrOPOE4jTuE/851Fzws07nTqsiLR5yrL3wsclOuEw91OB4lpSQcJFsVqMNpqu113GvA28GrQBnWcmjEeoCAjRUJTRTiWsu+5uRpoLBQjnJYOKiTNzXRxRPsGpjihcqCn31fCF4YJYJgJc1IFp+zyDY0TKSeJb5RV03I1V5Frc36y4qzCNwPN0rxQNCUz47Dg0PxWtRswYIISxScGYCKY6R2SGAtMlNkgx0FvqXmcoB+N0aecCqwy8VIjLCKzQqV5bIT2KvQ/IR7PhQaZOXurU70NLl51vcPu4eeD9vFJPfEmeAaegx3ggdfgGHwAp6AHiPXN+mH9tH41/thNe9NuzaQNq76zDW6E/fQvQMcTqg==</latexit> h 1 む StopGrad ⇥ + hg(f (x)), StopGrad(f (x ))i well-de ned む Q. fi 1 2 入 子 Chen, X. & He, K. (2021). Exploring simple siamese representation learning. 6 4 非 📄 (Non-contrastive learning) ⇤
  36. <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> [Tian-Chen-Ganguli x x0 <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x

    W 0 Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x 0 <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 8 4 📄 SimSiam ]
  37. <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> [Tian-Chen-Ganguli x x0 <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x

    W 0 Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x 0 <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x <latexit sha1_base64="qG5xXlYS9cXfNOcbLQsnyKaRsUE=">AAACZnicfVDRatswFFXcre3cLk1bxh72IhYKyQjBLqPbY+n20D1sa2FJCnEI18p1IirJRpJHg/E37Gv2un3H/mCfUTlNYU1GLwiOzjlXuvfEmeDGBsGfmrfx5Onm1vYzf2f3eX2vsX/QN2muGfZYKlJ9FYNBwRX2LLcCrzKNIGOBg/j6Q6UPvqM2PFXf7DzDkYSp4glnYB01brQjCXYWJ8VNOQ4iw+XizkAUX8pW0LlXP5XtcaMZdINF0XUQLkGTLOtivF87iiYpyyUqywQYMwyDzI4K0JYzgaUf5QYzYNcwxaGDCiSaUbHYqaRHjpnQJNXuKEsX7L8dBUhj5jJ2zmpGs6pV5P+0YW6T96OCqyy3qNjdR0kuqE1pFRCdcI3MirkDwDR3s1I2Aw3Muhh9P/qIbhmNn93DXzPUYFP9pohATyVXpVtuGnUq9JgRbu6NDvku2HA1xnXQP+6GJ92Ty7fN07NlxNvkFXlNWiQk78gpOScXpEcY+UF+kl/kd+2vV/deeC/vrF5t2XNIHpRHbwF9ALrR</latexit> <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x0 ⇠ N (0, I) 0 x, x ⇠ N (x0 , 2 I) 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 9 4 📄 SimSiam ]
  38. sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> <latexit 0 ⇠ x, ,x x x0 ⇠

    <latexit ( N x0 N <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> , 2 I) <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x0 (x 0, x <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x 2 I) [Tian-Chen-Ganguli <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> 0 x W 0 Stop Grad <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> 1 L( , W) = EkW x 2 <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x W x <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> StopGrad( 0 2 x )k2 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 0 5 📄 SimSiam ]
  39. <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit> encoder ˙ = predictor Ẇ = <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit>

    StopGrad( r L rW L 0 2 x )k2 ⇢ ⇢W ⇥ > > > 1 0 > r L= r E x W W x SG( x ) W x 2⇥ ⇤ > > > 0 > = E W W xx W xx <latexit sha1_base64="bYmQXGhvQmVRqMqW14vBkrQLrzk=">AAAEj3iclVNtb9MwEHZDgBHeVvjIF4sK1vFSNdW08QVUMRAggSiCrpPmrnJcJ7VmO5HtolVRfhC/hq/s3+CkXWm6tRqWLD2+e+7uudM5SDjTptk8qzjX3Os3bm7c8m7fuXvv/mb1wYGOx4rQLol5rA4DrClnknYNM5weJopiEXDaC072c3/vJ1WaxfKHmSS0L3AkWcgINtY0qDpvUUAjJlPMWSTpMPOQxAHHgxQJbEZBCFFnxLLiQTCHn72nryEKFSZ+azUzCNL3GeI0NEfeuff0GJk4WeSWDL3iBefPReI8BXw5JSiRfv+Q1S/lbG2X865N9H+61pf1kGLRyPQhQsWQVg2idwV9ZXELakvBeci/PrbKQUty6v5zpFkk8HFr+wpKVpWcOj1E5XC+MoPNWrPRLA68CPwZqIHZ6QyqlR00jMlYUGkIx1of+c3E9FOsDCOc2h0ca5pgcoIjemShxILqflpsewafWMsQhrGyVxpYWBcjUiy0nojAMnPNetmXGy/1BWKpsglf9VMmk7GhkkwLh2MOTQzzrwSHTFFi+MQCTBSz2iEZYfs1jP1wnofeUducol9soa8JVdjE6lmKsIoEk5ltNkIvcrSOiE/PiRbZOfvLU70IDloNf7ex+22n1t6fTXwDPAKPQR34YA+0wUfQAV1AnF/Ob+ePc+ZW3T33jdueUp3KLOYhKB3301+DlIMC</latexit> = (1 + 2 > )W W W x > ⇤ > > 0 W SG( x ) > 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 1 5 📄 1 L( , W) = EkW x 2 [Tian-Chen-Ganguli <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> ]
  40. 1 L( , W) = EkW x 2 <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit>

    encoder ˙ = predictor Ẇ = <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit> <latexit sha1_base64="lpvXaRnegHLEnsCAmPrXjC6upVs=">AAACa3icfZDfahNBFMYn67+6/mlq79SLwRAQkbBbSvWyWJHeFCOYtpCJ4ezkbDJ0dmadOVsMS96iT+OtvoQP0XfobJpCbcUDAz++883MOV9WauUpSf60ojt3791/sPYwfvT4ydP19sazQ28rJ3EgrbbuOAOPWhkckCKNx6VDKDKNR9nJXtM/OkXnlTVfaV7iqICpUbmSQEEat3uiAJplOf/EBX6v1Cm/EkR/pq7zN0G2HLc7SS9ZFr8N6Qo6bFX98UarKyZWVgUakhq8H6ZJSaMaHCmpcRGLymMJ8gSmOAxooEA/qpeLLXg3KBOeWxeOIb5Ur9+oofB+XmTB2Uzqb/Ya8V+9YUX5+1GtTFkRGnn5UV5pTpY3KfGJcihJzwOAdCrMyuUMHEgKWcax+IhhGYcH4eHPJTog697UAty0UGYRlpuKtw39zwg/royB4hBsejPG23C41Ut3ejtftju7H1YRr7EX7BV7zVL2ju2yfdZnAybZGfvJfrHfrfNoM3oevby0Rq3VnU32V0XdC6I2vDY=</latexit> F⌘ encoder predictor Ẇ = 2 x )k2 StopGrad( r L ⇢ (2 R rW L ⇢W (2 R h⇥d ) h⇥h ) <latexit sha1_base64="UE+ggd1yXB6jy2/M6xzgt1jVAi4=">AAACWHicfVBNT9tAEJ24H4D7FeDIZdUIiVZVZFcIekSUAxcERQ0gxWk03kySFeu1tTuuiCwf+mt6bX8O/BrWIZUoVH3SSk9v3uzMvLTQynEUXbeCJ0+fPV9aXglfvHz1+k17de3M5aWV1JO5zu1Fio60MtRjxZouCkuYpZrO08vPTf38O1mncvOVZwUNMpwYNVYS2UvD9sZWokySIU/TtDqtv1XThFVGTozqd8N2J+pGc4jHJF6QDixwMlxtbSajXJYZGZYanevHUcGDCi0rqakOk9JRgfISJ9T31KCfNKjmV9Ri0ysjMc6tf4bFXL3fUWHm3CxLvbPZ1z2sNeK/av2Sx58GlTJFyWTk3aBxqQXnoolEjJQlyXrmCUqr/K5CTtGiZB9cGCYH5I+xdOQ/Pi7IIuf2fZWgnWTK1P64SfKhYf8z4tUfo2ehDzZ+GONjcvaxG+90d75sd/b2FxEvwwa8hS2IYRf24BBOoAcSfsBP+AW/WzcBBEvByp01aC161uEvBGu34yK08w==</latexit> <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit> > anticommutator {A, B} = AB + BA <latexit sha1_base64="Tl7uGbE8x06oKCnvrCVx8gW1feI=">AAADG3icfVHLbtNAFB2bVzGvtCzZjIiKCmkiO0ItG6QKKsQGUSRSV8qk0Xh87Qy1Z6yZMSKy/Cl8DTvElgUbvoVxmrqviCuNdOaec99RkXFtfP+P4964eev2nbW73r37Dx4+6qxvHGpZKgYjJjOpjiKqIeMCRoabDI4KBTSPMgijk7cNH34FpbkUn828gElOU8ETzqixrmnnL4kg5aKiGU8FxLVHYmkqklMzixL8rsbPXuP+VtAjmqc5PR4+J+fk9hkKj4mRRfsjNe7hlTrL9PGQqJlsSUzI5ZrhipptfBvVO4f9i/lCj4CI23Gmna4/8BeGr4NgCbpoaQfTdWfTdsPKHIRhGdV6HPiFmVRUGc4ysPspNRSUndAUxhYKmoOeVItD1HjTemKcSGWfMHjhvRhR0VzreR5ZZdOvvso1zlXcuDTJq0nFRVEaEOy0UFJm2EjcXBXHXAEz2dwCyhS3vWI2o4oyY2/veWQf7DAKPtjEHwtQ1Ej1oiJUpTkXtR0uJdsN+p+QfjsTWuTZxQZX13gdHA4Hwc5g59PL7t6b5YrX0BP0FG2hAO2iPfQeHaARYs6+88XRjnG/uz/cn+6vU6nrLGMeo0vm/v4HOS75cQ==</latexit> Ḟ = 0 (1 + 2 (1 + 2 > ){F, W W} + {F, W} )WF + F 2⇢F ⇢W 1 2 h⇥h ) h⇥h ) (2 R <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit> Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 2 5 📄 [Tian-Chen-Ganguli <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> (2 R <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit> ]
  41. > <latexit sha1_base64="R0KJGfpm5TCJAQxKqWxQr51hEwM=">AAACbnicfVDRShtBFJ2s1dqt2qjgSykdDIIUCbulqCAFsUX60FILXSNk03B3chMHZ2eWmbvSsOQ7/Jq+tt/Qv+gndBK3YFW8MHA459yZOScrlHQURb8bwdyj+YXHi0/Cp0vLK8+aq2unzpRWYCKMMvYsA4dKakxIksKzwiLkmcJOdvFuqncu0Tpp9FcaF9jLYaTlUAogT/WbcYe/5Un60W8MoF91Jjz5lpIpdnh6wI9vase10m+2onY0G34XxDVosXpO+quNrXRgRJmjJqHAuW4cFdSrwJIUCidhWjosQFzACLseasjR9apZtgnf8syAD431RxOfsTc3KsidG+eZd+ZA5+62NiXv07olDfd7ldRFSajF9UPDUnEyfFoUH0iLgtTYAxBW+r9ycQ4WBPk6wzB9jz6MxU/+4s8FWiBjX1Up2FEu9cSHG6U7U/SQEb7/M3oU+mLj2zXeBaev2/Fue/fLm9bhUV3xInvONtk2i9keO2Qf2AlLmGBX7Af7yX41/gQbwYvg5bU1aNQ76+y/Cbb/Aiscu0w=</latexit> W = U ⇤ W U ,

    F = U ⇤F U 1 U̇ = O 0 2 L( , W) = EkW x StopGrad( x )k2 2 ; W <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> > <latexit sha1_base64="leMdFU7rYGLdDGrdzPyYDpyAzOQ=">AAACRXicfZDLSgMxFIYz3h3vunQTLIKIlBkRdSOIunAjKlgVO0XOpKc1mMkMyRmxDH0Lt/o6PoMP4U7caloreMMDgY///CfJ+eNMSUtB8OT19Q8MDg2PjPpj4xOTU9Mzs6c2zY3AikhVas5jsKikxgpJUnieGYQkVngWX+92+mc3aKxM9Qm1Mqwl0NSyIQWQky6iekq8wrf44eV0KSgH3eK/IexBifXq6HLGW3TTIk9Qk1BgbTUMMqoVYEgKhW0/yi1mIK6hiVWHGhK0taL75TZfdEqdN1LjjibeVb9OFJBY20pi50yAruzPXkf8q1fNqbFZK6TOckItPh5q5IpTyjv787o0KEi1HIAw0v2ViyswIMil5PvRHrplDB64iw8zNECpWS4iMM1E6rZbrhmtdOg/I9x+Gh35LtjwZ4y/4XS1HK6X14/XSts7vYhH2DxbYEssZBtsm+2zI1Zhgml2x+7Zg/foPXsv3uuHtc/rzcyxb+W9vQMXYbCg</latexit> <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit> encoder ˙ = predictor Ẇ = <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit> <latexit sha1_base64="lpvXaRnegHLEnsCAmPrXjC6upVs=">AAACa3icfZDfahNBFMYn67+6/mlq79SLwRAQkbBbSvWyWJHeFCOYtpCJ4ezkbDJ0dmadOVsMS96iT+OtvoQP0XfobJpCbcUDAz++883MOV9WauUpSf60ojt3791/sPYwfvT4ydP19sazQ28rJ3EgrbbuOAOPWhkckCKNx6VDKDKNR9nJXtM/OkXnlTVfaV7iqICpUbmSQEEat3uiAJplOf/EBX6v1Cm/EkR/pq7zN0G2HLc7SS9ZFr8N6Qo6bFX98UarKyZWVgUakhq8H6ZJSaMaHCmpcRGLymMJ8gSmOAxooEA/qpeLLXg3KBOeWxeOIb5Ur9+oofB+XmTB2Uzqb/Ya8V+9YUX5+1GtTFkRGnn5UV5pTpY3KfGJcihJzwOAdCrMyuUMHEgKWcax+IhhGYcH4eHPJTog697UAty0UGYRlpuKtw39zwg/royB4hBsejPG23C41Ut3ejtftju7H1YRr7EX7BV7zVL2ju2yfdZnAybZGfvJfrHfrfNoM3oevby0Rq3VnU32V0XdC6I2vDY=</latexit> F⌘ encoder predictor r L Ẇ = (2 R ⇢W (2 R <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit> rW L ) k[F, W ](t)kF  e 2 0 t k[F, W ](0)kF ! 0 <latexit sha1_base64="TrYHDSznIOFppgyhdEc/jxZYZb8=">AAACdXicfVBNTxRBEO0dUXH8WvBoSDquGDS4zhCCHIkQwsWIicuSbI+Tmt7apUNPz6S7xrgZ9q/4a7zC3V/ild5lSBSMlXTy8upVVb+XlVo5iqJfreDOwt179xcfhA8fPX7ytL20fOSKykrsyUIX9jgDh1oZ7JEijcelRcgzjf3sdHfW739D61RhvtCkxCSHsVEjJYE8lba3xdlgf72frNFrcZbuc6GR49f67YbQfskQ0ojT9FoTNRoqeJS2O1E3mhe/DeIGdFhTh+lSa1UMC1nlaEhqcG4QRyUlNVhSUuM0FJXDEuQpjHHgoYEcXVLPLU75qmeGfFRY/wzxOfvnRA25c5M888oc6MTd7M3If/UGFY22k1qZsiI08urQqNLce5zlxYfKoiQ98QCkVf6vXJ6ABUk+1TAUe+jNWPzoF38q0QIV9k0twI5zZabe3Fisz9D/hPD9WuhR6IONb8Z4GxxtdOOt7tbnzc7OhybiRfacvWBrLGbv2Q47YIesxyT7wX6yc3bR+h2sBC+DV1fSoNXMPGN/VfDuEsddvcU=</latexit> h⇥h ) > <latexit sha1_base64="Xyff2gcYItq/VGP+geKkuVJfHjw=">AAACTnicfZBBaxNBFMdno9V0rTaxx16GhkBsa9j10PYiLK2EXsQKJlvILuHt5G06dHZ2mZltDUu+hVev9et47RfxVnSSVNBGfDDw4//+M/PePykE18bzbp3ao8drT57W191nG89fbDaaLwc6LxXDPstFrs4T0Ci4xL7hRuB5oRCyRGCYXJ7M++EVKs1z+clMC4wzmEiecgbGSqNGszPs7YcxfUt7IX1Nw96rUaPldb1F0VXw76EV7ER7X26D6dmo6bSjcc7KDKVhArQe+l5h4gqU4UzgzI1KjQWwS5jg0KKEDHVcLWaf0bZVxjTNlT3S0IX6540KMq2nWWKdGZgL/bA3F//VG5YmPYorLovSoGTLj9JSUJPTeRB0zBUyI6YWgCluZ6XsAhQwY+Ny3egd2mUUvrcPfyhQgcnVbhWBmmRczuxyk2h/Tv8zwuffRkuuDdZ/GOMqDN50/YPuwUe/FRyTZdXJNtkhHeKTQxKQU3JG+oSRa/KV3JBvznfnh3Pn/Fxaa879nS3yV9XqvwDE8bQ2</latexit> ([F, W ] = F W 角 WF) anticommutator {A, B} = AB + BA <latexit sha1_base64="Tl7uGbE8x06oKCnvrCVx8gW1feI=">AAADG3icfVHLbtNAFB2bVzGvtCzZjIiKCmkiO0ItG6QKKsQGUSRSV8qk0Xh87Qy1Z6yZMSKy/Cl8DTvElgUbvoVxmrqviCuNdOaec99RkXFtfP+P4964eev2nbW73r37Dx4+6qxvHGpZKgYjJjOpjiKqIeMCRoabDI4KBTSPMgijk7cNH34FpbkUn828gElOU8ETzqixrmnnL4kg5aKiGU8FxLVHYmkqklMzixL8rsbPXuP+VtAjmqc5PR4+J+fk9hkKj4mRRfsjNe7hlTrL9PGQqJlsSUzI5ZrhipptfBvVO4f9i/lCj4CI23Gmna4/8BeGr4NgCbpoaQfTdWfTdsPKHIRhGdV6HPiFmVRUGc4ysPspNRSUndAUxhYKmoOeVItD1HjTemKcSGWfMHjhvRhR0VzreR5ZZdOvvso1zlXcuDTJq0nFRVEaEOy0UFJm2EjcXBXHXAEz2dwCyhS3vWI2o4oyY2/veWQf7DAKPtjEHwtQ1Ej1oiJUpTkXtR0uJdsN+p+QfjsTWuTZxQZX13gdHA4Hwc5g59PL7t6b5YrX0BP0FG2hAO2iPfQeHaARYs6+88XRjnG/uz/cn+6vU6nrLGMeo0vm/v4HOS75cQ==</latexit> Ḟ = ⇢ h⇥d <latexit sha1_base64="UE+ggd1yXB6jy2/M6xzgt1jVAi4=">AAACWHicfVBNT9tAEJ24H4D7FeDIZdUIiVZVZFcIekSUAxcERQ0gxWk03kySFeu1tTuuiCwf+mt6bX8O/BrWIZUoVH3SSk9v3uzMvLTQynEUXbeCJ0+fPV9aXglfvHz1+k17de3M5aWV1JO5zu1Fio60MtRjxZouCkuYpZrO08vPTf38O1mncvOVZwUNMpwYNVYS2UvD9sZWokySIU/TtDqtv1XThFVGTozqd8N2J+pGc4jHJF6QDixwMlxtbSajXJYZGZYanevHUcGDCi0rqakOk9JRgfISJ9T31KCfNKjmV9Ri0ysjMc6tf4bFXL3fUWHm3CxLvbPZ1z2sNeK/av2Sx58GlTJFyWTk3aBxqQXnoolEjJQlyXrmCUqr/K5CTtGiZB9cGCYH5I+xdOQ/Pi7IIuf2fZWgnWTK1P64SfKhYf8z4tUfo2ehDzZ+GONjcvaxG+90d75sd/b2FxEvwwa8hS2IYRf24BBOoAcSfsBP+AW/WzcBBEvByp01aC161uEvBGu34yK08w==</latexit> (1 + 2 (1 + 2 > ){F, W W} + {F, W} )WF + F 2⇢F ⇢W h⇥h ) h⇥h ) (2 R <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit> Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 3 5 📄 Point W, F (2 R <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit>
  42. 1 L( , W) = EkW x 2 ˙ =

    <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit> encoder predictor Ẇ = F⌘ encoder predictor ṗ = ⇢ (2 R ⇢W (2 R h⇥d ) h⇥h ) > 2 <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ṡ = 2 x )k2 <latexit sha1_base64="UE+ggd1yXB6jy2/M6xzgt1jVAi4=">AAACWHicfVBNT9tAEJ24H4D7FeDIZdUIiVZVZFcIekSUAxcERQ0gxWk03kySFeu1tTuuiCwf+mt6bX8O/BrWIZUoVH3SSk9v3uzMvLTQynEUXbeCJ0+fPV9aXglfvHz1+k17de3M5aWV1JO5zu1Fio60MtRjxZouCkuYpZrO08vPTf38O1mncvOVZwUNMpwYNVYS2UvD9sZWokySIU/TtDqtv1XThFVGTozqd8N2J+pGc4jHJF6QDixwMlxtbSajXJYZGZYanevHUcGDCi0rqakOk9JRgfISJ9T31KCfNKjmV9Ri0ysjMc6tf4bFXL3fUWHm3CxLvbPZ1z2sNeK/av2Sx58GlTJFyWTk3aBxqQXnoolEjJQlyXrmCUqr/K5CTtGiZB9cGCYH5I+xdOQ/Pi7IIuf2fZWgnWTK1P64SfKhYf8z4tUfo2ehDzZ+GONjcvaxG+90d75sd/b2FxEvwwa8hS2IYRf24BBOoAcSfsBP+AW/WzcBBEvByp01aC161uEvBGu34yK08w==</latexit> <latexit sha1_base64="vwl8TN6W0Oo1Blxxb0DOTkfwYEc=">AAACWHicfVBNT9tAEJ24LR9ugUCPXFaNkGiFIhsh6BHRHrhUpVUDSHEajTeTZMV6be2OKyLLB34N1/bn0F/TdUilFqo+aaWnN292Zl5aaOU4iu5awZOnz5aWV1bD5y/W1jfam1vnLi+tpJ7MdW4vU3SklaEeK9Z0WVjCLNV0kV69a+oX38g6lZsvPCtokOHEqLGSyF4atrd3E2WSDHmaptXn+ms1TVhl5MS0fj1sd6JuNId4TOIF6cACZ8PN1k4yymWZkWGp0bl+HBU8qNCykprqMCkdFSivcEJ9Tw36SYNqfkUtdrwyEuPc+mdYzNU/OyrMnJtlqXc2+7qHtUb8V61f8vjtoFKmKJmMvB80LrXgXDSRiJGyJFnPPEFpld9VyClalOyDC8PkPfljLH3wH38syCLn9k2VoJ1kytT+uEmy17D/GfH6t9Gz0AcbP4zxMTnf78aH3cNPB53jk0XEK7ANr2AXYjiCYziFM+iBhBu4he/wo/UzgGA5WL23Bq1Fz0v4C8HWL+qKtPc=</latexit> rW L <latexit sha1_base64="lpvXaRnegHLEnsCAmPrXjC6upVs=">AAACa3icfZDfahNBFMYn67+6/mlq79SLwRAQkbBbSvWyWJHeFCOYtpCJ4ezkbDJ0dmadOVsMS96iT+OtvoQP0XfobJpCbcUDAz++883MOV9WauUpSf60ojt3791/sPYwfvT4ydP19sazQ28rJ3EgrbbuOAOPWhkckCKNx6VDKDKNR9nJXtM/OkXnlTVfaV7iqICpUbmSQEEat3uiAJplOf/EBX6v1Cm/EkR/pq7zN0G2HLc7SS9ZFr8N6Qo6bFX98UarKyZWVgUakhq8H6ZJSaMaHCmpcRGLymMJ8gSmOAxooEA/qpeLLXg3KBOeWxeOIb5Ur9+oofB+XmTB2Uzqb/Ya8V+9YUX5+1GtTFkRGnn5UV5pTpY3KfGJcihJzwOAdCrMyuUMHEgKWcax+IhhGYcH4eHPJTog697UAty0UGYRlpuKtw39zwg/royB4hBsejPG23C41Ut3ejtftju7H1YRr7EX7BV7zVL2ju2yfdZnAybZGfvJfrHfrfNoM3oevby0Rq3VnU32V0XdC6I2vDY=</latexit> 0 StopGrad( r L <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit> 2(1 + (1 + 2 2 )p s + 2ps )ps + s ⇢p 2⇢s ODE decoupled 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 4 5 📄 [Tian-Chen-Ganguli <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> ]
  43. encoder 2 <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ṡ = predictor ṗ = 2(1

    + (1 + 2 2 )p s + 2ps )ps + s ⇢p 2⇢s <latexit sha1_base64="2bedRcYvj72EHD7BQRn2va20ldQ=">AAACX3icfZDPbhMxEMadpaVlW0oKJ9SLlahSy59oN4fCBSkqHHpBFIm0lbIhmnUmiVWvbdmziGiVOw/Ac/QKr9Jj36RO0krQIkay9NM3n8eeL7dKekqSy1r0YGX14dr6o3hj8/HWk/r20xNvSiewK4wy7iwHj0pq7JIkhWfWIRS5wtP8/P28f/oNnZdGf6GpxX4BYy1HUgAFaVBvZEND3PPXbbsgy98FztzE7AWR26/t/UG9mbSSRfH7kN5As9PIXv687EyPB9u13TBLlAVqEgq876WJpX4FjqRQOIuz0qMFcQ5j7AXUUKDvV4tlZnw3KEM+Mi4cTXyh/nmjgsL7aZEHZwE08Xd7c/FfvV5Jo7f9SmpbEmqxfGhUKk6Gz5PhQ+lQkJoGAOFk+CsXE3AgKOQXx9kHDMs4/BgGf7LogIx7UWXgxoXUs7DcOHs1p/8Z4futMVAcgk3vxngfTtqt9KB18Dltdg7ZstbZDmuwPZayN6zDjtgx6zLBfrAL9ov9rl1Fa9FWVF9ao9rNnWfsr4qeXwOeArf3</latexit> ṡ 2pṗ = 2⇢(s <latexit sha1_base64="sJ3TLRb9JYwiwJl/NdarhY7ea3o=">AAACVnicfVBdSxtBFJ3d+tWt1VifSkUGg1BKDbt5UF8KwfahL0WFRoVsDHcnN8ng7Owyc1caluCTP8VX++pPaX9D/0Nxkii0Kh4YOJx77syck+RKWgrD357/YmZ2bn7hZfBq8fXScmXlzZHNCiOwKTKVmZMELCqpsUmSFJ7kBiFNFB4nZ5/H8+NzNFZm+jsNc2yn0NeyJwWQkzqVt5Zv8fy0zj9x0QnxtNyqx2aQcRp1KtWwFk7AH5PojlQbazeHfy7Xbw46K95m3M1EkaImocDaVhTm1C7BkBQKR0FcWMxBnEEfW45qSNG2y0mGEd90Spf3MuOOJj5R/90oIbV2mCbOmQIN7MPZWHxq1iqot9supc4LQi2mD/UKxcmFdIXwrjQoSA0dAWGk+ysXAzAgyNUWBPEXdGEMfnMX7+dogDLzoYzB9FOpRy5cP/44Zs8Z4ce90bHAFRs9rPExOarXou3a9mFUbeyxKRbYO7bB3rOI7bAG+8oOWJMJdsGu2DX76f3y/vqz/vzU6nt3O6vsP/iVW6octtA=</latexit> (i.e. s = p 2 ) s NN 2 p ) 2 p = c0 e ⾒ s = p2 Predictor 2 <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> ṗ = p {1 (1 + 2 )p} ⇢p 方 1 2 方 立 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 5 5 📄 [Tian-Chen-Ganguli 2⇢t ]
  44. [Tian-Chen-Ganguli encoder 2 <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ṡ = predictor ṗ =

    2(1 + (1 + 2 (i.e. s = p 2 2 )p s + 2ps )ps + s 2⇢s ⇢p む2 ) む Q. む Q. Predictor 2 <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> ṗ = p {1 (1 + 2 )p} : <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> <latexit sha1_base64="85SJb0yUccJuZk+1ih/lgL1e73o=">AAACQ3icfZDLSgMxFIYz3h1vrS7dDBZBRMqMSHVZ1IUbUcFqoVPLmfR0GkwyQ5IRy9CXcKuv40P4DO7ErWBaK3jDA4GP//xJzvmjlDNtfP/JGRufmJyanpl15+YXFpcKxeULnWSKYo0mPFH1CDRyJrFmmOFYTxWCiDheRtcHg/7lDSrNEnlueik2BcSSdRgFY6V6qFks4Gq7VSj5ZX9Y3m8IRlAiozptFZ31sJ3QTKA0lIPWjcBPTTMHZRjl2HfDTGMK9BpibFiUIFA38+HAfW/dKm2vkyh7pPGG6tcbOQiteyKyTgGmq3/2BuJfvUZmOnvNnMk0Myjpx0edjHsm8Qbbe22mkBreswBUMTurR7uggBqbkeuGh2iXUXhsHz5JUYFJ1GYegooFk327XBxuDeg/I9x+Gi25NtjgZ4y/4WK7HFTKlbOdUnV/FPEMWSVrZIMEZJdUyRE5JTVCCSd35J48OI/Os/PivH5Yx5zRnRXyrZy3d/cXsKU=</latexit> 2 ( )& ⇢( <latexit sha1_base64="qka/CrKQLTtgrd32QEdkppKPDyw=">AAACP3icfZDNSgMxFIUz/jv+tbp0M1gEESkzItVlURduRAWrhU6RO+ltG5pJhiQjlqGv4FZfx8fwCdyJW3dmagW14oXAx7knyb0nSjjTxvefnYnJqemZ2bl5d2FxaXmlUFy90jJVFGtUcqnqEWjkTGDNMMOxniiEOOJ4HfWO8v71LSrNpLg0/QSbMXQEazMKJpdC1ZU3hZJf9ofljUMwghIZ1flN0dkMW5KmMQpDOWjdCPzENDNQhlGOAzdMNSZAe9DBhkUBMepmNhx24G1apeW1pbJHGG+ofr+RQax1P46sMwbT1b97ufhXr5Ga9kEzYyJJDQr6+VE75Z6RXr6512IKqeF9C0AVs7N6tAsKqLH5uG54jHYZhaf24bMEFRiptrMQVCdmYmCX64Q7Of1nhLsvoyXXBhv8jnEcrnbLQaVcudgrVQ9HEc+RdbJBtkhA9kmVnJBzUiOUdMk9eSCPzpPz4rw6b5/WCWd0Z438KOf9A97AryU=</latexit> p=0 ⇢p 方 自 1 2 目 方 自 自 立 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 6 5 📄 : ) ]
  45. <latexit sha1_base64="3GPTXDUNJzSzuzbIHbnhZyOHpNY=">AAACRnicfZBNaxRBEIZr1q84fiV69NK4BKLIMhMk5iIE9ZCLmICbBHaWUNNbs2nS09101wSXYX+G1+Tv+Bf8E96CV3s3G8iHWNDw8NZb3dVv6bQKnGW/ks6du/fuP1h6mD56/OTps+WV53vBNl5SX1pt/UGJgbQy1GfFmg6cJ6xLTfvl8adZf/+EfFDWfOOJo2GNY6MqJZGjNChGloX7UK2514fL3ayXzUvchnwBXVjUzuFKshrHZVOTYakxhEGeOR626FlJTdO0aAI5lMc4pkFEgzWFYTvfeSpWozISlfXxGBZz9epEi3UIk7qMzhr5KNzszcR/9QYNV5vDVhnXMBl58VDVaMFWzAIQI+VJsp5EQOlV3FXII/QoOcaUpsVnip/x9CVe/NWRR7b+TVugH9fKTOPnxsXbGf3PiN8vjZHSGGx+M8bbsLfeyzd6G7vvulsfFxEvwUt4BWuQw3vYgm3YgT5IsPADTuEs+Zn8Ts6TPxfWTrKYeQHXqgN/AYpPsF4=</latexit> ṗ = f (p) む む <latexit sha1_base64="eNGawypYh8jUPIrz6KQjBDmydQc=">AAACQ3icfZDLSgMxFIYz3h3vunQTLIKIlBkRdSOIunAjKlgtdIqcSU9rMJMZkjNiGfoSbvV1fAifwZ24FUxrBW94IPDxn/8kOX+cKWkpCJ68gcGh4ZHRsXF/YnJqemZ2bv7cprkRWBGpSk01BotKaqyQJIXVzCAkscKL+Hq/27+4QWNlqs+onWE9gZaWTSmAnFSNGinxbCe4nC0F5aBX/DeEfSixfp1cznnLblbkCWoSCqythUFG9QIMSaGw40e5xQzENbSw5lBDgrZe9D7c4ctOafBmatzRxHvq14kCEmvbSeycCdCV/dnrin/1ajk1t+uF1FlOqMXHQ81ccUp5d3vekAYFqbYDEEa6v3JxBQYEuYx8PzpAt4zBI3fxcYYGKDWrRQSmlUjdccu1orUu/WeE20+jI98FG/6M8Tecr5fDzfLm6UZpd68f8RhbZEtshYVsi+2yQ3bCKkwwxe7YPXvwHr1n78V7/bAOeP2ZBfatvLd3Sq2wSA==</latexit>

    ṗ = 0 ( ) む f (p) < 0: む f (p) > 0: <latexit sha1_base64="4CwN7wBSv4ewWRFHC5eS/mYDyFk=">AAACQXicfZDLSsNAFIYn9VbjXZdugkWoIiURUVci6sKNWMG2QlPKyfSkjk4mYWYiltB3cKuv41P4CO7ErRunF0GteGDg4z//zJzzBwlnSrvui5UbG5+YnMpP2zOzc/MLi0vLVRWnkmKFxjyWVwEo5ExgRTPN8SqRCFHAsRbcHvf6tTuUisXiUncSbETQFixkFLSRqmEx2Thwm4sFt+T2yxkFbwgFMqxyc8la91sxTSMUmnJQqu65iW5kIDWjHLu2nypMgN5CG+sGBUSoGll/3K6zbpSWE8bSHKGdvvr9RgaRUp0oMM4I9LX63euJf/XqqQ73GxkTSapR0MFHYcodHTu93Z0Wk0g17xgAKpmZ1aHXIIFqk5Bt+ydolpF4Zh4+T1CCjuVm5oNsR0x0zXJtf6tH/xnh/stoyDbBer9jHIXqdsnbLe1e7BQOj4YR58kqWSNF4pE9ckhOSZlUCCU35IE8kifr2Xq13qz3gTVnDe+skB9lfXwCDeuvKQ==</latexit> ・ 力 方 <latexit sha1_base64="RjerUXlalf2i8IQxl5ZkTsd6De4=">AAACQXicfZDLSsNAFIYn9VbjXZdugkWoIiURURcuRF24ESvYVmhKOZme1NHJJMxMxBL6Dm71dXwKH8GduHXj9CKoFQ8MfPznn5lz/iDhTGnXfbFyY+MTk1P5aXtmdm5+YXFpuariVFKs0JjH8ioAhZwJrGimOV4lEiEKONaC2+Nev3aHUrFYXOpOgo0I2oKFjII2UjUsJhsHbnOx4Jbcfjmj4A2hQIZVbi5Z634rpmmEQlMOStU9N9GNDKRmlGPX9lOFCdBbaGPdoIAIVSPrj9t11o3ScsJYmiO001e/38ggUqoTBcYZgb5Wv3s98a9ePdXhfiNjIkk1Cjr4KEy5o2Ont7vTYhKp5h0DQCUzszr0GiRQbRKybf8EzTISz8zD5wlK0LHczHyQ7YiJrlmu7W/16D8j3H8ZDdkmWO93jKNQ3S55u6Xdi53C4dEw4jxZJWukSDyyRw7JKSmTCqHkhjyQR/JkPVuv1pv1PrDmrOGdFfKjrI9PCjmvJw==</latexit> 7 5 :
  46. [Tian-Chen-Ganguli <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> 2 ṗ = p {1 (1 +

    2 )p} ⇢p strong weight decay: trivial solution p = 0 only <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> 自 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 8 5 📄 : ]
  47. [Tian-Chen-Ganguli <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> 2 ṗ = p {1 (1 +

    2 )p} ⇢p strong weight decay: trivial solution p = 0 only <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> 自 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 9 5 📄 : ]
  48. [Tian-Chen-Ganguli <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> 2 ṗ = p {1 (1 +

    2 )p} ⇢p <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> p=0 自 1 2 Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs. 0 6 📄 自 : ]
  49. ( ) ne-tuning む [Chen-He む [Bao ] ] :

    : fi 1 小 2 大 0 1 5 2 Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics. 3 6 📄 む CIFAR-
  50. [Tian+ ] <latexit sha1_base64="eD8Tr3fo5YqK6nI45PCr3QpXMzM=">AAADI3icjZJLj9MwEMed8FrCqwtHLhbVil1UqgShhQvSiofgsIgi6Haluqoc10mtdZzInqBWbj4On4Yb4sKBK58DJ82ifaAVI1n6aeY/4/GM40IKA2H40/MvXb5y9drG9eDGzVu373Q27x6YvNSMD1kuc30YU8OlUHwIAiQ/LDSnWSz5KD56VcdHX7g2IlefYVnwSUZTJRLBKDjXtPObZBTmjEq7X203HCdkMBe9lvFo58UaY/umIpInMH5MEk2ZJZKqVHL8V3ki/di36GECfAH2E+TFW01npy5p0S6qhzuY6KZcZcnqwoouvPrPmmRVES3SOUymnW7YDxvD5yFqoYtaG0w3vS0yy1mZcQVMUmPGUVjAxFINgrkuA1IaXlB2RFM+dqhoxs3ENgup8JbzzHCSa3cU4MZ7MsPSzJhlFjtl3a45G6ud/4qNS0ieT6xQRQlcsfVFSSkx5LjeLp4JzRnIpQPKtHC9Yjanblvg/kAQkNfcPUbz967wh4JrCrl+ZAnVaSZU5R6Xkl5NFwnp4ljoKHCDjc6O8TwcPOlHu/3dj0+7ey/bEW+g++gB2kYReob20Ds0QEPEvH1Pe9Zb+V/9b/53/8da6nttzj10yvxffwA3vAWu</latexit> SimSiam impl <latexit sha1_base64="9WSF9hO9iCGzJ0i5pz1CGM6hyoc=">AAADYHicfVJdb9MwFHUaYCV8bGVv8GJRIa1sqxI0DbSnSQPBC2JsdJtUt5XjOIk150O2g1pM/guv8I945ZfgtGnpuoorWTo69/ie6+vr55xJ5bq/rYZ95+69jeZ958HDR483t1pPLmRWCEJ7JOOZuPKxpJyltKeY4vQqFxQnPqeX/vVJlb/8SoVkWfpFTXI6SHCUspARrAw1alnbKGHpSCNZ+FJhcq1RglXshzA8gjPo67NyGECksjkBz4Y6LiFCcC6OjpZy8Q3tudHue2VZLsq9K0cLl7Gh99ZnhrvwO1zWcRqqvrOPOE4jTuE/851Fzws07nTqsiLR5yrL3wsclOuEw91OB4lpSQcJFsVqMNpqu113GvA28GrQBnWcmjEeoCAjRUJTRTiWsu+5uRpoLBQjnJYOKiTNzXRxRPsGpjihcqCn31fCF4YJYJgJc1IFp+zyDY0TKSeJb5RV03I1V5Frc36y4qzCNwPN0rxQNCUz47Dg0PxWtRswYIISxScGYCKY6R2SGAtMlNkgx0FvqXmcoB+N0aecCqwy8VIjLCKzQqV5bIT2KvQ/IR7PhQaZOXurU70NLl51vcPu4eeD9vFJPfEmeAaegx3ggdfgGHwAp6AHiPXN+mH9tH41/thNe9NuzaQNq76zDW6E/fQvQMcTqg==</latexit> min f

    :Rd !Rh g:Rh !Sh 1 Ex Ex+ |x 行 1 2 面 Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics. 4 6 📄 1 0 2 L( , W) = EkW x StopGrad( x )k2 2  0 hW x, StopGrad( x )i L( , W) = E 0 kW xkkStopGrad( x )k <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> ⇥ + hg(f (x)), StopGrad(f (x ))i ⇤
  51. <latexit sha1_base64="eD8Tr3fo5YqK6nI45PCr3QpXMzM=">AAADI3icjZJLj9MwEMed8FrCqwtHLhbVil1UqgShhQvSiofgsIgi6Haluqoc10mtdZzInqBWbj4On4Yb4sKBK58DJ82ifaAVI1n6aeY/4/GM40IKA2H40/MvXb5y9drG9eDGzVu373Q27x6YvNSMD1kuc30YU8OlUHwIAiQ/LDSnWSz5KD56VcdHX7g2IlefYVnwSUZTJRLBKDjXtPObZBTmjEq7X203HCdkMBe9lvFo58UaY/umIpInMH5MEk2ZJZKqVHL8V3ki/di36GECfAH2E+TFW01npy5p0S6qhzuY6KZcZcnqwoouvPrPmmRVES3SOUymnW7YDxvD5yFqoYtaG0w3vS0yy1mZcQVMUmPGUVjAxFINgrkuA1IaXlB2RFM+dqhoxs3ENgup8JbzzHCSa3cU4MZ7MsPSzJhlFjtl3a45G6ud/4qNS0ieT6xQRQlcsfVFSSkx5LjeLp4JzRnIpQPKtHC9Yjanblvg/kAQkNfcPUbz967wh4JrCrl+ZAnVaSZU5R6Xkl5NFwnp4ljoKHCDjc6O8TwcPOlHu/3dj0+7ey/bEW+g++gB2kYReob20Ds0QEPEvH1Pe9Zb+V/9b/53/8da6nttzj10yvxffwA3vAWu</latexit> L( , W) = E  0 hW

    x, StopGrad( x )i 0 kW xkkStopGrad( x )k む Taking derivative wrt normalizer makes gradient complicated む Solution: high-dimensional limit [Chen-He ] kW xk = const. 1 2 <latexit sha1_base64="VqeTNcrxlJFrSSdGNDLoE0QnlyY=">AAACbHicfZDfahNBFMYnW/+067+09q4Ig6EiIutsXNb2olDUC2/ECKYpZEKYnZxNhu7OLjNnJWGbx/BpeqsP4Uv4DM6mCaioBwZ+fOebmXO+pMyURca+t7ytGzdv3d7e8e/cvXf/QXt378wWlZHQl0VWmPNEWMiUhj4qzOC8NCDyJINBcvGm6Q8+g7Gq0J9wUcIoF1OtUiUFOmncfsEveS5wlqR0sAbem6mNNueXJxxhjrUstMVgOW53WMAidnQcUhZ0j6OIdR3E8cs4jGgYsFV1yLp6493WIZ8UsspBo8yEtcOQlTiqhUElM1j6vLJQCnkhpjB0qEUOdlSvNlvSQ6dMaFoYdzTSlfrrjVrk1i7yxDmbie2fvUb8W29YYXo0qpUuKwQtrz9Kq4xiQZuY6EQZkJgtHAhplJuVypkwQqIL0/f5W3DLGHjvHv5QghFYmGc1F2aaK710y03584b+ZxTzjdGR74LdpEf/DWfdIIyD+GPUOX29jnibHJDH5CkJyStySt6RHukTSb6QK/KVfGv98Pa9A+/RtdVrre88JL+V9+QnGtq9+Q==</latexit> 6 6 🔑 :
  52. r L ⇢ Ẇ = rW L ⇢W <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit>

    L loss [Tian+ 2 <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ṡ = 2(1 + ṗ = (1 + 2 ] 2 )p s + 2ps )ps + s Cosine loss 2⇢s ⇢p Adiabatic elimination Adiabatic elimination 6 <latexit sha1_base64="fIc2AH4mp14inm7PC5NN3idswE8=">AAACeXicfZDNbhMxFIWd4a8MP03LshtDVKlqSTSToJQFlSrCgg2iSKStlElGd5w7E6u2x7I9iGiUl+nTdAs7noUNThokaBFXsvTp3GNf35Npwa2Loh+N4M7de/cfbDwMHz1+8nSzubV9asvKMByyUpTmPAOLgiscOu4EnmuDIDOBZ9nFYNk/+4LG8lJ9dnONYwmF4jln4LyUNt8k09JRfdROcgOs7g7SmOpJnx7QQdrVkx5te+jpSXdRxweJ5YUEz+3EzEqq02Yr6kSrorchXkOLrOsk3Wrs+nmskqgcE2DtKI60G9dgHGcCF2FSWdTALqDAkUcFEu24Xq25oLtemdK8NP4oR1fqnzdqkNbOZeadEtzM3uwtxX/1RpXLX49rrnTlULHrQXklqCvpMjM65QaZE3MPwAz3f6VsBj4v55MNw+Qd+mUMfvAPf9RowJVmv07AFJKrhV+uSF4u6X9G+Prb6Cn0wcY3Y7wNp91O3O/0P71qHb9dR7xBdsgLskdickiOyXtyQoaEkUtyRb6R742fwfNgL9i/tgaN9Z1n5K8Ker8ALDO+0A==</latexit> 2 <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> 1 2 7 ṗ = p {1 2 6 ˙ = <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit> (1 + 2 )p} ⇢p ṗ = 3 2C1 p + C2 p 1+ 2 C3 p 2 ⇢p
  53. [Bao 6 <latexit sha1_base64="fIc2AH4mp14inm7PC5NN3idswE8=">AAACeXicfZDNbhMxFIWd4a8MP03LshtDVKlqSTSToJQFlSrCgg2iSKStlElGd5w7E6u2x7I9iGiUl+nTdAs7noUNThokaBFXsvTp3GNf35Npwa2Loh+N4M7de/cfbDwMHz1+8nSzubV9asvKMByyUpTmPAOLgiscOu4EnmuDIDOBZ9nFYNk/+4LG8lJ9dnONYwmF4jln4LyUNt8k09JRfdROcgOs7g7SmOpJnx7QQdrVkx5te+jpSXdRxweJ5YUEz+3EzEqq02Yr6kSrorchXkOLrOsk3Wrs+nmskqgcE2DtKI60G9dgHGcCF2FSWdTALqDAkUcFEu24Xq25oLtemdK8NP4oR1fqnzdqkNbOZeadEtzM3uwtxX/1RpXLX49rrnTlULHrQXklqCvpMjM65QaZE3MPwAz3f6VsBj4v55MNw+Qd+mUMfvAPf9RowJVmv07AFJKrhV+uSF4u6X9G+Prb6Cn0wcY3Y7wNp91O3O/0P71qHb9dR7xBdsgLskdickiOyXtyQoaEkUtyRb6R742fwfNgL9i/tgaN9Z1n5K8Ker8ALDO+0A==</latexit> 3 2C1 p + C2 p

    2 1+ ṗ = C3 p kW xk <latexit sha1_base64="5Hr53OTIAowhIyWIpjNH1R8x+lQ=">AAACXnicfZDfShtBFMYnq9V0W2vUm4I30wahlBJ2RdRLUS+8KUZojJAJ4ezkbDI4O7vMzErCmmufxlt9Fu98lM7GFfxTemDgx3e+mTnnizIpjA2Ch5q3sPhhabn+0f/0eeXLamNt/dykuebY4alM9UUEBqVQ2LHCSrzINEISSexGl0dlv3uF2ohU/bHTDPsJjJSIBQfrpEHjG7tmCdhxFNNuBaw9Fs/ahF0PGs2gFcyLvoewgiapqj1Yq22xYcrzBJXlEozphUFm+wVoK7jEmc9ygxnwSxhhz6GCBE2/mO8yo1tOGdI41e4oS+fqyxsFJMZMk8g5yxnN214p/qvXy2283y+EynKLij99FOeS2pSWwdCh0MitnDoAroWblfIxaODWxef77BjdMhp/u4dPM9RgU/2zYKBHiVAzt9yI/Srpf0aYPBsd+S7Y8G2M7+F8uxXutnbPdpoHh1XEdbJJvpMfJCR75ICckDbpEE5uyC25I/e1R2/JW/FWn6xerbqzQV6V9/UvSB23fQ==</latexit> 2 ⇢p kW xk <latexit sha1_base64="5Hr53OTIAowhIyWIpjNH1R8x+lQ=">AAACXnicfZDfShtBFMYnq9V0W2vUm4I30wahlBJ2RdRLUS+8KUZojJAJ4ezkbDI4O7vMzErCmmufxlt9Fu98lM7GFfxTemDgx3e+mTnnizIpjA2Ch5q3sPhhabn+0f/0eeXLamNt/dykuebY4alM9UUEBqVQ2LHCSrzINEISSexGl0dlv3uF2ohU/bHTDPsJjJSIBQfrpEHjG7tmCdhxFNNuBaw9Fs/ahF0PGs2gFcyLvoewgiapqj1Yq22xYcrzBJXlEozphUFm+wVoK7jEmc9ygxnwSxhhz6GCBE2/mO8yo1tOGdI41e4oS+fqyxsFJMZMk8g5yxnN214p/qvXy2283y+EynKLij99FOeS2pSWwdCh0MitnDoAroWblfIxaODWxef77BjdMhp/u4dPM9RgU/2zYKBHiVAzt9yI/Srpf0aYPBsd+S7Y8G2M7+F8uxXutnbPdpoHh1XEdbJJvpMfJCR75ICckDbpEE5uyC25I/e1R2/JW/FWn6xerbqzQV6V9/UvSB23fQ==</latexit> <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> 自 5 2 自 Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics. 8 6 非 📄 自 : p=0 ]
  54. ⾒ [Simon+ Pn 1 <latexit sha1_base64="yJFc/F9o1bNbsh19fthasx/n+no=">AAACh3icfZDLahRBFIZr2lvS3ia6dFMYAioydrsYswkkKiKCGNFJAlOTprrmdKdIXZqq0zJNM4/iO/g0gitduvYJrJ7JgCbigYKP//xVdc6fV0p6TJJvvejS5StXr62tx9dv3Lx1u79x58Db2gkYCausO8q5ByUNjFCigqPKAde5gsP89EXXP/wEzktrPmJTwUTz0shCCo5ByvpvGMIMPTYKKNMcT/KCsg+y1JzuUFY4LlLDfK2zVu6k82Oz8jSZXOEsk8cMbZX1N5NBsih6EdIz2Nx9mLz6+evz3n620dtiUytqDQaF4t6P06TCScsdSqFgHrPaQ8XFKS9hHNBwDX7SLpae062gTGlhXTgG6UL980bLtfeNzoOzG9Sf73Xiv3rjGovtSStNVSMYsfyoqBVFS7sE6VQ6EKiaAFw4GWal4oSHpDDkHMfsJYRlHLwND7+rwHG07lHLuCu1NPOwXMked/Q/I5+tjIHiEGx6PsaLcPB0kA4Hw/ch4edkWWvkHrlPHpCUPCO75DXZJyMiyBfylXwnP6L16Ek0jLaX1qh3ducu+auivd9YaMrb</latexit> ⌃= n > y

    x i=1 i i <latexit sha1_base64="jCNnjMpT3TZMYK6q9miJ72KmV9k=">AAAConicfZHfahQxFMaz4786/tvqpTfBtdhqWWZ6UUUobK2oCGKLblvYbIcz2TPb0EkyJJnSZZhH8Yn6BL6Bt3qpN2Z2V1hb8UDgx3e+JCdf0iIX1kXR11Zw5eq16zeWboa3bt+5e6+9fH/f6tJw7HOda3OYgsVcKOw74XI8LAyCTHM8SE92mv7BKRortPrsJgUOJYyVyAQH56WkjUyCO04z9hakhJdblGUGeFxtqJrZUiaV2IrrI7U6t9GzRFQL/KQ+Yk4X9BldFBd42l5L2p2oG02LXoZ4Dp3eWvTm248v27vJcmuFjTQvJSrHc7B2EEeFG1ZgnOA51iErLRbAT2CMA48KJNphNc2jpiteGdFMG7+Uo1N1cUcF0tqJTL2zmdRe7DXiv3qD0mUvhpVQRelQ8dlFWZlTp2kTLh0Jg9zlEw/AjfCzUn4MPlDnvyAM2Wv0jzH4wR/8sUADTpunFQMzlkLV/nFjtt7Q/4xw9sfoKfTBxhdjvAz7G914s7u5F3d6r8islshD8oiskpg8Jz3yjuySPuHknHwnP8mv4HHwPtgLPs2sQWu+5wH5qwL2G1eq1Pg=</latexit> 1 := 2n ] Pn 0> 0 > (x x + x i xi ) i=1 i i 3 2 行 行 己 力 己 力 Simon, J. B., Knutins, M., Ziyin, L., Geisz, D., Fetterman, A. J., & Albrecht, J. (2023). On the stepwise nature of self-supervised learning. 9 自 6 自 入 📄 入 ⾒ (revisited)
  55. む (double descent) 古典的なトレードオフは ここだけを見ていた! 過剰パラメータ系で バリアンスが減少 方 Rocks, J.,

    & Mehta, P. (2022). Memorizing without overfitting: Bias, variance, and interpolation in overparameterized models. 5 7 📄 二 む
  56. 20 ] <latexit sha1_base64="a9CL46wQJecOLlRym7UAyvNYtog=">AAACZ3icfZDfShtBFMYna2tt2mq0IAVvpg0FKxp2i2hvCoK98EZqodGAG8PZyUkcnD/LzNnWsOSiT+NtfRwfoW/RSVzBRumBgd985xvOnC/LlfQUxze1aO7J0/lnC8/rL16+WlxqLK8ce1s4gW1hlXWdDDwqabBNkhR2coegM4Un2cX+pH/yA52X1nynUY5dDUMjB1IABanXWPvJP/P1zllKNuedD2flVjKubqNeoxm34mnxh5BU0GRVHfWWa9tp34pCoyGhwPvTJM6pW4IjKRSO62nhMQdxAUM8DWhAo++W0y3G/H1Q+nxgXTiG+FS9/6IE7f1IZ8Gpgc79bG8iPtrL9MxkGnzqltLkBaERt4MHheJk+SQi3pcOBalRABBOhr9zcQ4OBIUg6/X0C4blHB6GQV9zdEDWbZQpuKGWZhyWHaabE/qfES7vjIFCzslsqg/h+GMr2WntfNtu7u1XiS+wNfaOrbOE7bI9dsCOWJsJ9otdsd/suvYnWopWoze31qhWvXnN/qno7V8PrroG</latexit> 50 [ ] > w =

    (X X) 1 ( > ) <latexit sha1_base64="z2Dc08s5xAXt8BWmvZ1QQES6++M=">AAACZXicfZDRShtBFIYnq1aN2kZbBOmFQ4NoSw27IraXgl54oWihUcEN4ezkZB2cmV1mzpqGJfRpvNXn8Qn6Gp3ECG0sPTDw8Z//cOb8Sa6kozB8rART0zOvZufmqwuLS6/f1JZXzl1WWIFNkanMXibgUEmDTZKk8DK3CDpReJHcHAz7F7donczMd+rn2NKQGtmVAshL7dpabzNOkRzvbcdIEBtIFPDjrd7Hdq0eNsJR8ZcQjaHOxnXWXq7sxp1MFBoNCQXOXUVhTq0SLEmhcFCNC4c5iBtI8cqjAY2uVY5uGPANr3R4N7P+GeIj9c+JErRzfZ14pwa6dpO9ofjPXqInNlP3a6uUJi8IjXha3C0Up4wPA+IdaVGQ6nsAYaX/OxfXYEGQj7FajQ/RH2fxxC86zdECZfZTGYNNtTQDf2wafx7S/4zw49noyeccTab6Es53GtFeY+/bbn3/YJz4HHvPPrAtFrEvbJ8dsTPWZIL9ZHfsnj1UfgVLwbtg9ckaVMYzb9lfFaz/BkxmubM=</latexit> w X y 行 [ 6 7 : む 0 w ⌘rL(w)
  57. : む <latexit sha1_base64="mHwqfM0W8YU5KmEzLdFd5TAEMio=">AAAClnicfZBtSxwxEMdz2wft2YezfVPwTehROIscu0W0byyClZai9Ao9Fcx1mc3NrsEkuyRZ9Vj2O/XTFHxVP0qz50nrWToQ+GXmP8zMPymksC4ML1vBvfsPHi4sPmovPX7y9Fln+fmBzUvDcchzmZujBCxKoXHohJN4VBgElUg8TE53mvrhGRorcv3NTQocKci0SAUH51Nx5/NejylwJ0landerW8yWKq7EVlR/1wyl7DEJOpNI/4jW6CQWN9+L2rOZSlbjTjfsh9OgdyGaQZfMYhAvt9bZOOelQu24BGuPo7BwowqME1xi3WalxQL4KWR47FGDQjuqpkfX9LXPjGmaG/+0o9Ps3x0VKGsnKvHKZlk7X2uS/6wlam6yS9+NKqGL0qHm14PTUlKX08ZROhYGuZMTD8CN8LtTfgIGuPO+t9vsA/rjDO77QV8KNOBy86ZiYDIldO2PzdhaQ/8TwsWN0JP3OZp39S4cvO1HG/2Nr+vd7Z2Z44tkhbwiPRKRTbJNPpEBGRJOfpCf5Be5Cl4G74Pd4OO1NGjNel6QWxEMfgNbvs5J</latexit> む L(w) = hw⇤ , yi

    xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> n X i=1 `(m) = e <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) ; m <latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) = dt む ( rL(w(t)) ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> ŵ <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ŵ w(t) lim = t!1 kw(t)k kŵk minw2Rd kwk 2 s.t. 8i : hw, yi xi i 1 一 大 二 8 1 Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. 3 8 📄 [Srebro+ ]
  58. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> ŵ <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit>

    ŵ w(t) lim = t!1 kw(t)k kŵk minw2Rd kwk 2 s.t. 8i : hw, yi xi i 1 yi x i <latexit sha1_base64="DtGYJwzS8y9N9JprVZsArKmi2KE=">AAACW3icfZBNS8NAEIa38avGj1bFk5dgEUSkJFLUo1APXkQFq4IpYbKd1KW7m7C7EUvoL/GqP8qD/8VNraBVHFh4eOddZuaNM8608f23ijMzOze/UF10l5ZXVmv1tfUbneaKYoemPFV3MWjkTGLHMMPxLlMIIuZ4Gw/aZf/2EZVmqbw2wwy7AvqSJYyCsVJUrw0jFgowD3FSPI0iFtUbftMfl/cbggk0yKQuo7VKK+ylNBcoDeWg9X3gZ6ZbgDKMchy5Ya4xAzqAPt5blCBQd4vx5iNvxyo9L0mVfdJ4Y/X7jwKE1kMRW2e5pJ7uleKfvVhMTTbJcbdgMssNSvo5OMm5Z1KvjMXrMYXU8KEFoIrZ3T36AAqoseG5bniK9jiF53bQRYYKTKr2ihBUXzA5ssf2w/2S/jPC05fRks05mE71N9wcNIPD5uFVq3HSniReJVtkm+ySgByRE3JGLkmHUJKTZ/JCXivvzozjOsufVqcy+bNBfpSz+QEyUrcB</latexit> ŵ <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> 大 8 1 Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. 4 8 📄 [Srebro+ ]
  59. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> ŵ <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit>

    ŵ w(t) lim = t!1 kw(t)k kŵk minw2Rd kwk 2 s.t. 8i : hw, yi xi i 1 <latexit sha1_base64="2beUvCkyvZB/RdhsLSZwXSLFBHA=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVg/37ip1oNGMCj6G8Ih1MmwTm9q3k7USVkuUVkmwJjrMMhsuwBtORPYr0S5wQzYHXTx2qECiaZdDJbu0zWndGiSaveUpQP1+48CpDE9GTtnuaQZ7ZXin71Yjky2yX674CrLLSr2OTjJBbUpLROhHa6RWdFzAExztztlt6CBWZdbpRIdojtO47EbdJKhBpvqzSIC3ZVc9d2x3WirpP+M8PhldORyDkdT/Q0X241wt7F7tlNvHgwTnyYrZJWsk5DskSY5IqekRRjJyBN5Ji/eq/fuT/hTn1bfG/5ZJj/Kr34ADoa1dA==</latexit> w(T ) <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) ŵ <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> 大 8 1 Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. 5 8 📄 [Srebro+ ]
  60. ] : w(t) = r(t) · u(t) む <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit>

    <latexit sha1_base64="BTfHnp/5rWUjw9t97GIzTG5w70I=">AAACnnicfZBdaxNBFIYn61eNH0310pvBIKRSw66U6k2hUBEvLK1i2kAnhrOTs9uhM7PLzFlNXPK//CveeKs/w9k0FU3FAwMP73sOZ86bllp5iuNvreja9Rs3b63dbt+5e+/+emfjwbEvKidxIAtduGEKHrWyOCBFGoelQzCpxpP0fL/xTz6h86qwH2hW4shAblWmJFCQxp33b3vCAJ2lGf+8yXe5IJySp5lGLnxlxrXaTeYfrcBp2XsmNNi8cS4ntn7jl7ESbuFujjvduB8vil+FZAldtqyj8UZrW0wKWRm0JDV4f5rEJY1qcKSkxnlbVB5LkOeQ42lACwb9qF4cP+dPgjLhWeHCs8QX6p8TNRjvZyYNnc1f/arXiP/0UrOymbKXo1rZsiK08mJxVmlOBW+S5RPlUJKeBQDpVPg7l2fgQFLIv90WrzAc5/AgLDos0QEV7mktwOVG2Xk4NhdbDf2vEaaXjYFCzslqqlfh+Hk/2envvNvu7u0vE19jj9hj1mMJe8H22Bt2xAZMsq/sO/vBfkY8eh0dRIcXrVFrOfOQ/VXR8Bdfq8/B</latexit> L(w) = i=1 exp( hw, zi i) ; zi := yi xi <latexit sha1_base64="CYBsIAf6ejlzLwUOo9fdk5WPTTo=">AAACaXicfZDfahNBFMZnt2rb+C+pN0UpDAZBpIRdCWkRhEK98EaM0KSFbljOTs4mQ2Zml5nZ0rjkyqfxVp+mz9CXcDbZgKbFAwM/vvMN55wvyQU3NghuPH/rwcNH2zu7jcdPnj573mztDU1WaIYDlolMXyRgUHCFA8utwItcI8hE4HkyO63651eoDc/UmZ3nOJIwUTzlDKyT4uZBJMFOk5R+jzn98JHOY75WrmMeN9tBJ1gWvQthDW1SVz9ued1onLFCorJMgDGXYZDbUQnaciZw0YgKgzmwGUzw0qECiWZULu9Y0DdOGdM00+4pS5fq3z9KkMbMZeKc1Y5ms1eJ9/YSuTHZpsejkqu8sKjYanBaCGozWoVEx1wjs2LuAJjmbnfKpqCBWRdloxF9Qnecxi9u0NccNdhMvysj0BPJ1cIdO4kOK/qfEa7XRkcu53Az1bswfN8Je53et2775LROfIe8Iq/JWxKSI3JCPpM+GRBGfpCf5Bf57d36LX/ff7my+l795wX5p/z2H13vu5w=</latexit> kzi k  1 <latexit sha1_base64="VszzYOV0URl+yCZaxJDyP37+WKk=">AAACYHicfVBNSwMxEE3Xr1q/WvWkl2ARRKTsiqhHQQ9eRAWrglvKbDpbg0l2SbJiXftfvOo/8uovMVsraBUfBF7evGFmXpQKbqzvv5W8sfGJyanydGVmdm5+oVpbvDRJphk2WSISfR2BQcEVNi23Aq9TjSAjgVfR3WFRv7pHbXiiLmwvxZaEruIxZ2Cd1K4uh0+hBHsbxfSxzd1HIA3a1brf8Aegv0kwJHUyxFm7VtoJOwnLJCrLBBhzE/ipbeWgLWcC+5UwM5gCu4Mu3jiqQKJp5YP1+3TdKR0aJ9o9ZelA/d6RgzSmJyPnLFY1o7VC/LMWyZHJNt5v5VylmUXFPgfHmaA2oUU2tMM1Mit6jgDT3O1O2S1oYNYlWKmER+iO03jiBp2mqMEmejMPQXclV313bDfcKth/Rnj4Mjrmcg5GU/1NLrcbwW5j93ynfnA4TLxMVska2SAB2SMH5JickSZh5JE8kxfyWnr3yt6CV/u0eqVhzxL5AW/lAx/HuEo=</latexit> <latexit sha1_base64="bqBQXDCDWA+nPdlWVC4j8BeWJhg=">AAACZHicfZDRShtBFIYna2ttam1UCgVBBkMhKZLuFrHeCIK98MKiglHBDeHs5GwcnJldZs62DUv6NL1t36cv0OdwNkbQKD0w8PGffzjn/EmupKMw/FsL5p49n3+x8LL+avH10pvG8sqZyworsCsyldmLBBwqabBLkhRe5BZBJwrPk+v9qn/+Da2TmTmlUY49DUMjUymAvNRvvDtsxRroKkn59xa123yXH7Wij9TuN5phJ5wUfwzRFJpsWsf95dpWPMhEodGQUODcZRTm1CvBkhQKx/W4cJiDuIYhXno0oNH1yskJY/7eKwOeZtY/Q3yi3v9RgnZupBPvrNZ1s71KfLKX6JnJlO70SmnygtCI28FpoThlvMqHD6RFQWrkAYSVfncursCCIJ9ivR5/QX+cxa9+0FGOFiizH8oY7FBLM/bHDuPNiv5nhB93Rk8+52g21cdw9qkTbXe2T7aae/vTxBfYGttgLRaxz2yPHbBj1mWC/WS/2G/2p/YvWAxWg7e31qA2/bPKHlSwfgOFNLfM</latexit> L(w(t)) = O(1/t) r(t) = ⇥(log t) span({zi }) <latexit sha1_base64="gEzgC3pGdxe+436igLwyn6ffUdY=">AAACanicfVBdSxtBFJ1dbdXtV7RPIsjQtGBLCbsi2kdBH3wRFYwJuCHcndxNhszMLjOzYrrsW3+Nr/pn/A/+CGdjhDYWDwycOfdc7r0nyQU3NgzvPX9h8c3bpeWV4N37Dx8/NVbXLkxWaIZtlolMdxMwKLjCtuVWYDfXCDIR2EnGB3W9c4Xa8Eyd20mOPQlDxVPOwDqp39iMJdiRlqXJQVVbcTn9Jyn93edx9b3faIatcAr6kkQz0iQznPZXvZ14kLFCorJMgDGXUZjbXgnaciawCuLCYA5sDEO8dFSBRNMrp4dU9JtTBjTNtHvK0qn6d0cJ0piJTJyzXtPM12rxv7VEzk226a9eyVVeWFTsaXBaCGozWqdEB1wjs2LiCDDN3e6UjUADsy7LIIgP0R2n8dgNOslRg830jzIGPZRcVe7YYfyzZq8Z4frZ6JjLOZpP9SW52G5Fu63ds53m/sEs8WWyQb6QLRKRPbJPjsgpaRNG/pAbckvuvAd/zV/3N56svjfr+Uz+gf/1EVTtvRA=</latexit> 8 : Pn <latexit sha1_base64="CfNqES2E21yVkH2Ja8Pg7riXN/I=">AAACYXicfZBdSxtBFIYna1tj+pUoXnkzNBRiKWG3iHojCPbCG6kFo4IbwtnJyWbIfCwzZ0vDkh/jrf4ir/tHnMQU2lh6YODhPe9wznmzQklPcfxQi9ZevHy1Xt9ovH7z9t37Zmvz0tvSCewJq6y7zsCjkgZ7JEnhdeEQdKbwKpuczPtXP9B5ac0FTQvsa8iNHEkBFKRBc9t1aJcf8fRijASdVNmc0+6g2Y678aL4c0iW0GbLOh+0anvp0IpSoyGhwPubJC6oX4EjKRTOGmnpsQAxgRxvAhrQ6PvVYv8Z/xiUIR9ZF54hvlD//FGB9n6qs+DUQGO/2puL/+xlemUyjQ77lTRFSWjE0+BRqThZPg+HD6VDQWoaAISTYXcuxuBAUIiw0Ui/YjjO4VkY9K1AB2TdpyoFl2tpZuHYPP08p/8Z4edvY6CQc7Ka6nO4/NJN9rv73/faxyfLxOtsh31gHZawA3bMTtk56zHBKnbL7th97Ve0ETWjzSdrVFv+2WJ/VbTzCKTlt38=</latexit> 1 む Step 方 : 方 む Step 2 : 1 む Step 0 : 3 む Step 大 大 む 6 8 [Soudry+ む ; active
  61. L(t) = n X i=1 <latexit sha1_base64="P7vLWTVN7CZCE6uXK9LQM0PkbYI=">AAADSHicjVJNb9QwEHVSPsry0S0cuViskAoqqwRVbS+VKpUDBxBFYttK9TaaeCdZt44T2Q6wpLlzhZ/FP+BfcENIHHDSpaLbBTFSpJc3b+aNx44LKYwNgq+ev3Dl6rXrizc6N2/dvrPUXb67Z/JScxzwXOb6IAaDUigcWGElHhQaIYsl7scnO01+/y1qI3L1xk4KHGaQKpEIDtZRUfcnO2UKYgn0xYp9xE7pFmUSE+toU2ZRJbbC+kjhUfWESVCpRMoysOM4oe+cfvX870MkmG4F9QVKpGPbdm3az2tNWaKBV391mGNQV22DY/ofRcfnRXPGirq9oB+0QS+DcAp6ZBq70bK3xkY5LzNUlksw5jAMCjusQFvBnUmHlQYL4CeQ4qGDCjI0w6q9ppo+dMyIJrl2n7K0Zf+sqCAzZpLFTtnMamZzDTk3F2czzjbZHFZCFaVFxc+Mk1JSm9PmDdCR0MitnDgAXAs3O+VjcNdg3UvpdNgzdIfT+NIZvSpQg83144qBTjOhanfYlK026F9CeP9b6JDbczi71ctg72k/XO+vv17rbe9MN75I7pMHZIWEZINsk+dklwwI92Lvo/fJ++x/8b/53/0fZ1Lfm9bcIxdiwf8FUuEQ6A==</latexit> exp( hw(t), zi

    i) <latexit sha1_base64="ebrEJfSGKzxfK5aTUbND9tbsZfo=">AAACg3icfVBdaxNBFJ2sVmv8aKqPvgwGoX4QdktofQkU6kMfFCuYttCJy93J3c3Qmdll5q4Y1/wUf42v9Qf4b5xNI2gqXrhwOOdc7r0nq7TyFMc/O9GNmxu3bm/e6d69d//BVm/74YkvaydxLEtdurMMPGplcUyKNJ5VDsFkGk+zi8NWP/2EzqvSfqB5hRMDhVW5kkCBSnv7ozc79IwLjTmJr8LXJm3UKFl8tAJ0NYNUBVkYoFmW8y+pEk4Vs+BMe/14EC+LXwfJCvTZqo7T7c5QTEtZG7QkNXh/nsQVTRpwpKTGRVfUHiuQF1DgeYAWDPpJs/xwwZ8GZsrz0oW2xJfsnxMNGO/nJgvO9la/rrXkP7XMrG2m/NWkUbaqCa28WpzXmlPJ2/j4VDmUpOcBgHQq3M7lDBxICiF3u+I1huccvg2L3lXogEr3vBHgCqPsIjxbiJct+p8RPv82BhRyTtZTvQ5OdgfJ3mDv/bB/cLhKfJM9Zk/YDkvYPjtgR+yYjZlk39h3dsl+RBvRi2g3Gl5Zo85q5hH7q6LRL7qAxbQ=</latexit> krL(t)k = n X e hw(t),zi i zi = L(t) i=1 n X L̇(t) = e hw(t),zi i i=1 <latexit sha1_base64="Uz3PxTUp6gaxbfxTmD0LQGPodm4=">AAACdnicfVDRShtBFJ2sbdXY1qiPggwNopUadkXUR0EffKjUglHBTcPdyc06ODO7zNwVw5J/6Nf0VX/DP/Gxs0kKbSxeGDhzzrlz554kV9JRGD7Vgpk3b9/Nzs3XF95/+LjYWFq+cFlhBbZFpjJ7lYBDJQ22SZLCq9wi6EThZXJ7VOmXd2idzMw5DXLsaEiN7EsB5KluY2v76yZ9/rHDY4U87mXEq/voth2noDV4bWzpNpphKxwVfwmiCWiySZ11l2q7/kVRaDQkFDh3HYU5dUqwJIXCYT0uHOYgbiHFaw8NaHSdcrTUkK97psf7mfXHEB+xf3eUoJ0b6MQ7NdCNm9Yq8r9aoqcmU/+gU0qTF4RGjAf3C8Up41VivCctClIDD0BY6f/OxQ1YEORzrdfjY/TLWTz1g77laIEyu1XGYFMtzdAvm8ZfKvSaEe7/GD3yOUfTqb4EFzutaK+19323eXg0SXyOrbJPbJNFbJ8dshN2xtpMsJ/sF3tgj7XnYC1YDzbG1qA26Vlh/1QQ/gaGKb4h</latexit> 2 2 言 一 言 0 L(t)  L̇(t)  e j z = L(t) i hw(t),z i j =: αi(t) n X ↵i (t)zi i=1 γ ≤ ∥Zα∥ ≤ 1 |`0 (·)| . `(·) <latexit sha1_base64="FKnMI/NRfyI3fAHsoYUydug33L0=">AAACb3icfVDbahRBFOydeInrLYkPPgjSuIiJyNAj2Y2+BeJDXoIR3CSQWcKZ3rObJn0Zus+ELJP9AL/GV/0UP8M/sGczAQ1qQUNRpw7Vp4pSq0BC/OgkS7du37m7fK97/8HDR49XVtcOgqu8xKF02vmjAgJqZXFIijQelR7BFBoPi7OdZn54jj4oZz/TrMSRgalVEyWBonSy0rvMUetX67kcO9q45LnGEIIyvJFbNbpE2n8fscVFOhBCZP2WiD7PUrFAj7XYP1ntbOZjJyuDlqSGEI4zUdKoBk9Kapx38ypgCfIMpngcqQWDYVQvrpnzl1EZ84nz8VniC/X3jRpMCDNTRKcBOg03Z43411lhbiTT5N2oVrasCK28Cp5UmpPjTVV8rDxK0rNIQHoV/87lKXiQFAvtdvMPGI/zuBeDPpbogZx/Xefgp0bZeTx2mr9p2P+McHFtjCz2fF0m/zc5eJtmg3TwabO3vdM2vsyesRdsnWVsi22zXbbPhkyyL+wr+8a+d34mT5PnCb+yJp125wn7A8nGL1w4vmQ=</latexit> hẇ(t), zi i = L(t) e P hw(t),zi i : self-bounding self-bounding property <latexit sha1_base64="aI1SRW4FfkgIkUf2hYJq03lQpY8=">AAAC1XicfVHbahRBEO0Zb3G9bRSfBG1chCjJMhNC4ksgEB98UIzgZgPp3aWmt2bSpLtn6O6JrpN5E8Env8+v8BfsmWyCboIFDaerTtWpS1JIYV0U/QrCa9dv3Ly1dLtz5+69+w+6yw/3bV4ajgOey9wcJGBRCo0DJ5zEg8IgqETiMDnebeLDEzRW5PqTmxU4UpBpkQoOzrsm3R9smjv6bsW9pNt0jdlSTSqxHddjTXFcrTEJOpNImQJ3lKT0syeunn++TgQzbby+4Plq1QW5bspeQW+lTpmGREKrzU7H65NuL+pHrdHLIJ6DHpnb3mQ52PByvFSoHZdg7WEcFW5UgXGC+546rLRYAD+GDA891KDQjqp2ZzV94T1TmubGP+1o6/07owJl7Uwlntn0bxdjjfPKWKIWlF36elQJXZQONT8TTktJXU6bg9CpMMidnHkA3AjfO+VHYIA7f7ZOh71BP5zB917oQ4EGXG5eVQxMpoSu/bAZW23Q/4jw5Zzokd9zvLjVy2B/vR9v9jc/bvR2ducbXyJPyHOyQmKyRXbIW7JHBoST38Hj4GnwLByGdfgt/H5GDYN5ziPyj4U//wB8NeF3</latexit> n X i=1 L  krLk  L <latexit sha1_base64="FRiVxzXG0ouxmD/o+wjDZiJZVw4=">AAACa3icfZDfShtBFMYna7Wa+ieaO1twaBBEJOwWUS+F9MILSy0YE3BCODs52QyZmV1mZsWw5tKn8bY+jA/hO3QSU2ijeGDgN9/5hjPnizMprAvDp1Kw8GFx6ePySvnT6tr6RmVz68qmueHY5KlMTTsGi1JobDrhJLYzg6Biia142Jj0WzdorEj1pRtl2FGQaNEXHJyXupUdloBSQM8pk0jZHdMQS39ld1PhvFuphfVwWvQ1RDOokVlddDdLh6yX8lyhdlyCtddRmLlOAcYJLnFcZrnFDPgQErz2qEGh7RTTTcZ01ys92k+NP9rRqfrviwKUtSMVe6cCN7DzvYn4Zi9Wc5Nd/6RTCJ3lDjV/GdzPJXUpncREe8Igd3LkAbgR/u+UD8AAdz7Mcpl9R7+cwR9+0M8MDbjU7BcMTKKEHvtlE3YwofeMcPvX6MnnHM2n+hquvtWjo/rRr8PaaWOW+DL5TL6SPRKRY3JKzsgFaRJO7skD+U0eS89BNdgOvrxYg9LsTZX8V8HuH9k2u8E=</latexit> 7 8 <latexit sha1_base64="7UuNyefxwLyJB6CgK0pFR3Lpe5I=">AAACj3icfVBdSxtBFJ1sbbXph0l97MvQIMRiw66I9sUSSB8stNSCUcFJl7uTu+vgzOwyM9sal/yg/hpf9d84G2NpY+mBgTPnnsu99ySFFNaF4U0jeLT0+MnyytPms+cvXq622q+ObF4ajkOey9ycJGBRCo1DJ5zEk8IgqETicXI+qOvHP9BYketDNylwpCDTIhUcnJfi1uBz123QPUqZLVVcib1o+l0zvCi675gEnUmkTIE7S1L60zs3f/8uY8HMzLARtzphL5yBPiTRnHTIHAdxu7HNxjkvFWrHJVh7GoWFG1VgnOASp01WWiyAn0OGp55qUGhH1ezaKV33ypimufFPOzpT/+yoQFk7UYl31rvaxVot/rOWqIXJLn0/qoQuSoea3w1OS0ldTuso6VgY5E5OPAFuhN+d8jMwwJ0PvNlkH9EfZ/CLH/S1QAMuN28rBiZTQk/9sRnbrNn/jHBxb/TM5xwtpvqQHG31op3ezrftTn8wT3yFvCZvSJdEZJf0yT45IEPCyS9yRa7JTdAOdoMPQf/OGjTmPWvkLwSfbgEM2sit</latexit> 2 2 krL(t)k <latexit sha1_base64="RM6GLvk33BZPeHaWeGtUn79D8SA=">AAACV3icfZBRSxtBEMf3zqoxrdXoY1+WhkIUSe9E1BdB0Ic+KFpoVPAOmdtM4uLu3rk7J4Yjn8PX9mP5aXQTI9hYOrDw4z//ZWb+WaGkoyh6DMKZD7Nz87WF+sdPi5+XlhsrZy4vrcCOyFVuLzJwqKTBDklSeFFYBJ0pPM9uDkb98zu0TubmFw0KTDX0jexJAeSl9KhFa3yPn7Ti77R2tdyM2tG4+HuIJ9Bkkzq9agRbSTcXpUZDQoFzl3FUUFqBJSkUDutJ6bAAcQN9vPRoQKNLq/HWQ/7NK13ey61/hvhYffujAu3cQGfeqYGu3XRvJP6zl+mpydTbTStpipLQiJfBvVJxyvkoEt6VFgWpgQcQVvrdubgGC4J8cPV6coj+OIvHftBJgRYot+tVAravpRn6Y/vJxoj+Z4T7V6Mnn3M8nep7ONtsx9vt7Z9bzf2DSeI19oV9ZS0Wsx22z36wU9Zhgt2yB/ab/Qkeg6dwLqy9WMNg8meV/VVh4xk237R2</latexit> L(t) = O(1/t) (Step )
  62. L(t) = n X i=1 <latexit sha1_base64="P7vLWTVN7CZCE6uXK9LQM0PkbYI=">AAADSHicjVJNb9QwEHVSPsry0S0cuViskAoqqwRVbS+VKpUDBxBFYttK9TaaeCdZt44T2Q6wpLlzhZ/FP+BfcENIHHDSpaLbBTFSpJc3b+aNx44LKYwNgq+ev3Dl6rXrizc6N2/dvrPUXb67Z/JScxzwXOb6IAaDUigcWGElHhQaIYsl7scnO01+/y1qI3L1xk4KHGaQKpEIDtZRUfcnO2UKYgn0xYp9xE7pFmUSE+toU2ZRJbbC+kjhUfWESVCpRMoysOM4oe+cfvX870MkmG4F9QVKpGPbdm3az2tNWaKBV391mGNQV22DY/ofRcfnRXPGirq9oB+0QS+DcAp6ZBq70bK3xkY5LzNUlksw5jAMCjusQFvBnUmHlQYL4CeQ4qGDCjI0w6q9ppo+dMyIJrl2n7K0Zf+sqCAzZpLFTtnMamZzDTk3F2czzjbZHFZCFaVFxc+Mk1JSm9PmDdCR0MitnDgAXAs3O+VjcNdg3UvpdNgzdIfT+NIZvSpQg83144qBTjOhanfYlK026F9CeP9b6JDbczi71ctg72k/XO+vv17rbe9MN75I7pMHZIWEZINsk+dklwwI92Lvo/fJ++x/8b/53/0fZ1Lfm9bcIxdiwf8FUuEQ6A==</latexit> exp( hw(t), zi

    i) <latexit sha1_base64="ebrEJfSGKzxfK5aTUbND9tbsZfo=">AAACg3icfVBdaxNBFJ2sVmv8aKqPvgwGoX4QdktofQkU6kMfFCuYttCJy93J3c3Qmdll5q4Y1/wUf42v9Qf4b5xNI2gqXrhwOOdc7r0nq7TyFMc/O9GNmxu3bm/e6d69d//BVm/74YkvaydxLEtdurMMPGplcUyKNJ5VDsFkGk+zi8NWP/2EzqvSfqB5hRMDhVW5kkCBSnv7ozc79IwLjTmJr8LXJm3UKFl8tAJ0NYNUBVkYoFmW8y+pEk4Vs+BMe/14EC+LXwfJCvTZqo7T7c5QTEtZG7QkNXh/nsQVTRpwpKTGRVfUHiuQF1DgeYAWDPpJs/xwwZ8GZsrz0oW2xJfsnxMNGO/nJgvO9la/rrXkP7XMrG2m/NWkUbaqCa28WpzXmlPJ2/j4VDmUpOcBgHQq3M7lDBxICiF3u+I1huccvg2L3lXogEr3vBHgCqPsIjxbiJct+p8RPv82BhRyTtZTvQ5OdgfJ3mDv/bB/cLhKfJM9Zk/YDkvYPjtgR+yYjZlk39h3dsl+RBvRi2g3Gl5Zo85q5hH7q6LRL7qAxbQ=</latexit> krL(t)k = n X ẇ(t) = e hw(t),zi i zi = L(t) i=1 rL(t) <latexit sha1_base64="d27rWqTcGStGblc62IeRjh5FZ44=">AAACbXicfVBdSxtBFJ1srdXY1tjiU0WmDaUqbdgtor4UBH3oQ0WFRgU3hLuTu3FwZnaZuasNS579Nb62v6W/on/B2RihRvHCwJlzz/06Sa6kozD8WwueTT2ffjEzW597+er1fGPhzZHLCiuwLTKV2ZMEHCppsE2SFJ7kFkEnCo+T850qf3yB1snM/KRBjh0NfSNTKYA81W28j3sZlbEGOktSfjlcoVX+jX+JDSQK+A//7TaaYSscBX8IojFosnEcdBdq676pKDQaEgqcO43CnDolWJJC4bAeFw5zEOfQx1MPDWh0nXJ0y5B/9EyPp5n1zxAfsf9XlKCdG+jEK6ul3WSuIh/NJXpiMqVbnVKavCA04nZwWihOGa+M4j1pUZAaeADCSr87F2dgQZC3s16Pd9EfZ3HPD9rP0QJldq2Mwfa1NEN/bD/+XKGnhPDrTuiR9zmadPUhOPraijZaG4frze2dseMz7B37wFZYxDbZNvvODlibCXbFrtlv9qf2L1gMloLlW2lQG9e8Zfci+HQDW2S8aA==</latexit> n X i=1 hw(t),zi i e P e j z = L(t) i hw(t),z i j =: αi(t) <latexit sha1_base64="g19DYhnvpUMDXiE2nziN4Qedxzc=">AAACg3icfVBdSxtBFJ1sa6vR1qiPvgwNBT9K2JWgfREEfehLqYJRwQ3h7uRuHJyZXWbutoZlf0p/ja/6A/w3nY0RNEovDJw599yvk+RKOgrDh0bw7v3ch4/zC83FpU+fl1srq2cuK6zAnshUZi8ScKikwR5JUniRWwSdKDxPrg/r/PlvtE5m5pTGOfY1jIxMpQDy1KC1Fw8zKmMNdJWk/E+1QZt8n9ckt09swbf957muqAatdtgJJ8Ffg2gK2mwax4OVRte3EIVGQ0KBc5dRmFO/BEtSKKyaceEwB3ENI7z00IBG1y8nF1b8q2eGPM2sf4b4hH1eUYJ2bqwTr6xXdLO5mnwzl+iZyZR+75fS5AWhEY+D00JxynhtHx9Ki4LU2AMQVvrdubgCC4K8yc1mfIT+OIs//aBfOVqgzG6VMdiRlqbyx47ibzX6nxBunoQeeZ+jWVdfg7OdTrTb2T3ptg8Op47Ps3X2hW2wiO2xA/aDHbMeE+wvu2V37D6YC7aDnaD7KA0a05o19iKC/X9j3MUA</latexit> <latexit sha1_base64="ceY1klkK67Ll+4yb87pgAaBH7vk=">AAACrHicfVFdaxQxFM2OX3X96FYffQkuQpW6zEipvhQK9cEXsUK3LWym453snWlokhmSO+o67N/zP/gffNVnM9st6LZ4IXBy7rmcm5O81spTHP/oRTdu3rp9Z+1u/979Bw/XBxuPjnzVOIljWenKneTgUSuLY1Kk8aR2CCbXeJyf73f948/ovKrsIc1qTA2UVhVKAgUqG3zaFb4xWat2k/mpxdP2pdBgS41cGKCzvOBfNun51uXlW6aEW/Tnq7rmel02GMajeFH8KkiWYMiWdZBt9LbFtJKNQUtSg/eTJK4pbcGRksG3LxqPNchzKHESoAWDPm0XUcz5s8BMeVG5cCzxBfv3RAvG+5nJg7Jb1a/2OvLaXm5WnKl4k7bK1g2hlRfGRaM5VbzLmU+VQ0l6FgBIp8LuXJ6BA0nhN/p98RbD4xy+D0YfanRAlXvRCnClUXYeHluKrQ79TwhfL4UBhZyT1VSvgqNXo2RntPNxe7i3v0x8jT1hT9kmS9hrtsfesQM2ZpJ9Zz/ZL/Y7GkWH0SRKL6RRbznzmP1TUfEHyAnWRw==</latexit> = n X i=1 <latexit sha1_base64="H28B7NZxjlH3pQP51/Hqla2B7n8=">AAACg3icfVBdaxNBFJ2sVmv8SuujL4NBqFrCbgmtL4VCffBBsYJpi50Q7k7ubobOxzJzVxqW/BR/ja/6A/w3zqYRNBUvDHM491zuPSevtAqUpj87ya3bG3fubt7r3n/w8NHj3tb2aXC1lziSTjt/nkNArSyOSJHG88ojmFzjWX553PbPvqAPytlPNK9wbKC0qlASKFKT3sHhux16ITTYUiMXBmiWF7yO3C7/LHKnp2Fu4idAVzNopX4pnfT66SBdFr8JshXos1WdTLY6QzF1sjZoSWoI4SJLKxo34ElJjYuuqANWIC+hxIsILRgM42bpcMGfR2bKC+fjs8SX7J8TDZjQHhqVrYWw3mvJf/Zys7aZitfjRtmqJrTyenFRa06Ot/HxqfIoSc8jAOlVvJ3LGXiQFEPudsUbjOY8vo+LPlTogZx/2QjwpVF2Ec2WYrdF/xPC1W9hRDHnbD3Vm+B0b5DtD/Y/DvtHx6vEN9lT9oztsIwdsCP2lp2wEZPsK/vGvrMfyUbyKtlLhtfSpLOaecL+quTwFxr+xOU=</latexit> ↵i (t)zi i=1 hu(t), rL(t)i ṙ(t) = ẇ(t) = ṙu + ru̇ hu, ẇi = ṙ + rhu, u̇i n X γ ≤ ∥Zα∥ ≤ 1 <latexit sha1_base64="1halCfcGzBByYr9cJKDTh3/GJEc=">AAACgnicfVBdaxNBFJ2sVWv8aKqPvgwNQi017LYl9kGhUB98aLGCaQudEO5O7m6HzswuM3eLYck/8df4qn/Af+NskkKbihcGzpx7Lufek5ZaeYrjP63owcrDR49Xn7SfPnv+Yq2z/vLUF5WTOJCFLtx5Ch61sjggRRrPS4dgUo1n6dVh0z+7RudVYb/RpMShgdyqTEmgQI06fTEuiLtNess/8ndCg801cmGALtOMV4Hf5sJCqoEfhY9wc8Go04178az4fZAsQJct6mS03toLTrIyaElq8P4iiUsa1uBISY3Ttqg8liCvIMeLAC0Y9MN6duCUvwnMmGeFC88Sn7G3J2ow3k9MGpTN5n6515D/7KVmyZmy/WGtbFkRWjk3zirNqeBNenysHErSkwBAOhV25/ISHEgKGbfb4hOG4xweB6MvJTqgwm3VAlxulJ2GY3Ox3aD/CeH7jTCgkHOynOp9cLrTS/q9/te97sHhIvFV9pptsE2WsPfsgH1mJ2zAJPvBfrJf7He0Em1FSbQ7l0atxcwrdqeiD38B0aPCyA==</latexit> <latexit sha1_base64="q+7ncnZrRwYGFSWoOE6wUV89GWg=">AAACq3ichVFNbxMxEHW2fJSUj7QcuViNKiEo0S6qCpdKlcqBC6IVpK2oo2jWmd1atb0re5YSrfLz+iP6G7jCHW8+pJIiOpKl55n3PDPPaamVpzi+bkUr9+4/eLj6qL32+MnTZ531jWNfVE5iXxa6cKcpeNTKYp8UaTwtHYJJNZ6kFwdN/eQ7Oq8K+5XGJQ4M5FZlSgKF1LAzFBpsrpELA3SeZrza5mJUUL24X06EmzH2pgXu+Gvu7lJVC9Ww04178TT4bZDMQZfN43C43toJL8nKoCWpwfuzJC5pUIMjJTVO2qLyWIK8gBzPArRg0A/qqRMTvhUyI54VLhxLfJq9qajBeD82aWA2k/rlWpP8Zy01S50pez+olS0rQitnjbNKcyp4YzMfKYeS9DgAkE6F2bk8BweSwme02+IDhuUcfgqNPpfogAr3qhbgcqPsJCybi+0G/Y8IPxbEgILPybKrt8Hx216y29s92unuH8wdX2Uv2CZ7yRL2ju2zj+yQ9ZlkV+wn+8V+R2+iL9G3SMyoUWuuec7+igj/AJQJ1U4=</latexit> e hw(t),zi i hu(t), zi i = L(t)hu(t), Z↵(t)i 1 <latexit sha1_base64="RYJ/+JLtDWjbLEB/6j1ZlmO6AJo=">AAACaHicfVBNSyNBEO2Mu37Er+gelkUPjWFBFw0zIupR0IOHlXXBqOCEUNOpxMbunqG7RgxDLvtrvLr/xr/gr7AnRtC4bEHD61ev+nW9JFPSURg+VoKJT58np6ZnqrNz8wuLtaXlc5fmVmBTpCq1lwk4VNJgkyQpvMwsgk4UXiQ3h2X/4hatk6k5o36GLQ09I7tSAHmqXVvd+rlOGzxWyONOSty+3kq6XauHjXBY/COIRqDORnXaXqrs+FdErtGQUODcVRRm1CrAkhQKB9U4d5iBuIEeXnloQKNrFcM1Bvy7Zzq8m1p/DPEh+3aiAO1cXydeqYGu3XivJP/ZS/SYM3X3W4U0WU5oxItxN1ecUl5mxDvSoiDV9wCElf7vXFyDBUE+yWo1PkK/nMUTb/QrQwuU2h9FDLanpRn4ZXvxZon+J4S7V6FHPudoPNWP4Hy7Ee02dn/v1A8OR4lPsxW2xtZZxPbYATtmp6zJBPvD7tkD+1t5CmrB1+DbizSojGa+sHcVrD0D5mC5Xw==</latexit> 8 8 <latexit sha1_base64="7UuNyefxwLyJB6CgK0pFR3Lpe5I=">AAACj3icfVBdSxtBFJ1sbbXph0l97MvQIMRiw66I9sUSSB8stNSCUcFJl7uTu+vgzOwyM9sal/yg/hpf9d84G2NpY+mBgTPnnsu99ySFFNaF4U0jeLT0+MnyytPms+cvXq622q+ObF4ajkOey9ycJGBRCo1DJ5zEk8IgqETicXI+qOvHP9BYketDNylwpCDTIhUcnJfi1uBz123QPUqZLVVcib1o+l0zvCi675gEnUmkTIE7S1L60zs3f/8uY8HMzLARtzphL5yBPiTRnHTIHAdxu7HNxjkvFWrHJVh7GoWFG1VgnOASp01WWiyAn0OGp55qUGhH1ezaKV33ypimufFPOzpT/+yoQFk7UYl31rvaxVot/rOWqIXJLn0/qoQuSoea3w1OS0ldTuso6VgY5E5OPAFuhN+d8jMwwJ0PvNlkH9EfZ/CLH/S1QAMuN28rBiZTQk/9sRnbrNn/jHBxb/TM5xwtpvqQHG31op3ezrftTn8wT3yFvCZvSJdEZJf0yT45IEPCyS9yRa7JTdAOdoMPQf/OGjTmPWvkLwSfbgEM2sit</latexit> L(t)  ṙ(t)  L(t) <latexit sha1_base64="CfNqES2E21yVkH2Ja8Pg7riXN/I=">AAACYXicfZBdSxtBFIYna1tj+pUoXnkzNBRiKWG3iHojCPbCG6kFo4IbwtnJyWbIfCwzZ0vDkh/jrf4ir/tHnMQU2lh6YODhPe9wznmzQklPcfxQi9ZevHy1Xt9ovH7z9t37Zmvz0tvSCewJq6y7zsCjkgZ7JEnhdeEQdKbwKpuczPtXP9B5ac0FTQvsa8iNHEkBFKRBc9t1aJcf8fRijASdVNmc0+6g2Y678aL4c0iW0GbLOh+0anvp0IpSoyGhwPubJC6oX4EjKRTOGmnpsQAxgRxvAhrQ6PvVYv8Z/xiUIR9ZF54hvlD//FGB9n6qs+DUQGO/2puL/+xlemUyjQ77lTRFSWjE0+BRqThZPg+HD6VDQWoaAISTYXcuxuBAUIiw0Ui/YjjO4VkY9K1AB2TdpyoFl2tpZuHYPP08p/8Z4edvY6CQc7Ka6nO4/NJN9rv73/faxyfLxOtsh31gHZawA3bMTtk56zHBKnbL7th97Ve0ETWjzSdrVFv+2WJ/VbTzCKTlt38=</latexit> r(t) = ⇥(log t) (Step )
  63. ] : w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む

    <latexit sha1_base64="BTfHnp/5rWUjw9t97GIzTG5w70I=">AAACnnicfZBdaxNBFIYn61eNH0310pvBIKRSw66U6k2hUBEvLK1i2kAnhrOTs9uhM7PLzFlNXPK//CveeKs/w9k0FU3FAwMP73sOZ86bllp5iuNvreja9Rs3b63dbt+5e+/+emfjwbEvKidxIAtduGEKHrWyOCBFGoelQzCpxpP0fL/xTz6h86qwH2hW4shAblWmJFCQxp33b3vCAJ2lGf+8yXe5IJySp5lGLnxlxrXaTeYfrcBp2XsmNNi8cS4ntn7jl7ESbuFujjvduB8vil+FZAldtqyj8UZrW0wKWRm0JDV4f5rEJY1qcKSkxnlbVB5LkOeQ42lACwb9qF4cP+dPgjLhWeHCs8QX6p8TNRjvZyYNnc1f/arXiP/0UrOymbKXo1rZsiK08mJxVmlOBW+S5RPlUJKeBQDpVPg7l2fgQFLIv90WrzAc5/AgLDos0QEV7mktwOVG2Xk4NhdbDf2vEaaXjYFCzslqqlfh+Hk/2envvNvu7u0vE19jj9hj1mMJe8H22Bt2xAZMsq/sO/vBfkY8eh0dRIcXrVFrOfOQ/VXR8Bdfq8/B</latexit> L(w) = i=1 exp( hw, zi i) ; zi := yi xi <latexit sha1_base64="CYBsIAf6ejlzLwUOo9fdk5WPTTo=">AAACaXicfZDfahNBFMZnt2rb+C+pN0UpDAZBpIRdCWkRhEK98EaM0KSFbljOTs4mQ2Zml5nZ0rjkyqfxVp+mz9CXcDbZgKbFAwM/vvMN55wvyQU3NghuPH/rwcNH2zu7jcdPnj573mztDU1WaIYDlolMXyRgUHCFA8utwItcI8hE4HkyO63651eoDc/UmZ3nOJIwUTzlDKyT4uZBJMFOk5R+jzn98JHOY75WrmMeN9tBJ1gWvQthDW1SVz9ued1onLFCorJMgDGXYZDbUQnaciZw0YgKgzmwGUzw0qECiWZULu9Y0DdOGdM00+4pS5fq3z9KkMbMZeKc1Y5ms1eJ9/YSuTHZpsejkqu8sKjYanBaCGozWoVEx1wjs2LuAJjmbnfKpqCBWRdloxF9Qnecxi9u0NccNdhMvysj0BPJ1cIdO4kOK/qfEa7XRkcu53Az1bswfN8Je53et2775LROfIe8Iq/JWxKSI3JCPpM+GRBGfpCf5Bf57d36LX/ff7my+l795wX5p/z2H13vu5w=</latexit> kzi k  1 <latexit sha1_base64="VszzYOV0URl+yCZaxJDyP37+WKk=">AAACYHicfVBNSwMxEE3Xr1q/WvWkl2ARRKTsiqhHQQ9eRAWrglvKbDpbg0l2SbJiXftfvOo/8uovMVsraBUfBF7evGFmXpQKbqzvv5W8sfGJyanydGVmdm5+oVpbvDRJphk2WSISfR2BQcEVNi23Aq9TjSAjgVfR3WFRv7pHbXiiLmwvxZaEruIxZ2Cd1K4uh0+hBHsbxfSxzd1HIA3a1brf8Aegv0kwJHUyxFm7VtoJOwnLJCrLBBhzE/ipbeWgLWcC+5UwM5gCu4Mu3jiqQKJp5YP1+3TdKR0aJ9o9ZelA/d6RgzSmJyPnLFY1o7VC/LMWyZHJNt5v5VylmUXFPgfHmaA2oUU2tMM1Mit6jgDT3O1O2S1oYNYlWKmER+iO03jiBp2mqMEmejMPQXclV313bDfcKth/Rnj4Mjrmcg5GU/1NLrcbwW5j93ynfnA4TLxMVska2SAB2SMH5JickSZh5JE8kxfyWnr3yt6CV/u0eqVhzxL5AW/lAx/HuEo=</latexit> <latexit sha1_base64="bqBQXDCDWA+nPdlWVC4j8BeWJhg=">AAACZHicfZDRShtBFIYna2ttam1UCgVBBkMhKZLuFrHeCIK98MKiglHBDeHs5GwcnJldZs62DUv6NL1t36cv0OdwNkbQKD0w8PGffzjn/EmupKMw/FsL5p49n3+x8LL+avH10pvG8sqZyworsCsyldmLBBwqabBLkhRe5BZBJwrPk+v9qn/+Da2TmTmlUY49DUMjUymAvNRvvDtsxRroKkn59xa123yXH7Wij9TuN5phJ5wUfwzRFJpsWsf95dpWPMhEodGQUODcZRTm1CvBkhQKx/W4cJiDuIYhXno0oNH1yskJY/7eKwOeZtY/Q3yi3v9RgnZupBPvrNZ1s71KfLKX6JnJlO70SmnygtCI28FpoThlvMqHD6RFQWrkAYSVfncursCCIJ9ivR5/QX+cxa9+0FGOFiizH8oY7FBLM/bHDuPNiv5nhB93Rk8+52g21cdw9qkTbXe2T7aae/vTxBfYGttgLRaxz2yPHbBj1mWC/WS/2G/2p/YvWAxWg7e31qA2/bPKHlSwfgOFNLfM</latexit> L(w(t)) = O(1/t) r(t) = ⇥(log t) span({zi }) <latexit sha1_base64="gEzgC3pGdxe+436igLwyn6ffUdY=">AAACanicfVBdSxtBFJ1dbdXtV7RPIsjQtGBLCbsi2kdBH3wRFYwJuCHcndxNhszMLjOzYrrsW3+Nr/pn/A/+CGdjhDYWDwycOfdc7r0nyQU3NgzvPX9h8c3bpeWV4N37Dx8/NVbXLkxWaIZtlolMdxMwKLjCtuVWYDfXCDIR2EnGB3W9c4Xa8Eyd20mOPQlDxVPOwDqp39iMJdiRlqXJQVVbcTn9Jyn93edx9b3faIatcAr6kkQz0iQznPZXvZ14kLFCorJMgDGXUZjbXgnaciawCuLCYA5sDEO8dFSBRNMrp4dU9JtTBjTNtHvK0qn6d0cJ0piJTJyzXtPM12rxv7VEzk226a9eyVVeWFTsaXBaCGozWqdEB1wjs2LiCDDN3e6UjUADsy7LIIgP0R2n8dgNOslRg830jzIGPZRcVe7YYfyzZq8Z4frZ6JjLOZpP9SW52G5Fu63ds53m/sEs8WWyQb6QLRKRPbJPjsgpaRNG/pAbckvuvAd/zV/3N56svjfr+Uz+gf/1EVTtvRA=</latexit> 8 : Pn <latexit sha1_base64="CfNqES2E21yVkH2Ja8Pg7riXN/I=">AAACYXicfZBdSxtBFIYna1tj+pUoXnkzNBRiKWG3iHojCPbCG6kFo4IbwtnJyWbIfCwzZ0vDkh/jrf4ir/tHnMQU2lh6YODhPe9wznmzQklPcfxQi9ZevHy1Xt9ovH7z9t37Zmvz0tvSCewJq6y7zsCjkgZ7JEnhdeEQdKbwKpuczPtXP9B5ac0FTQvsa8iNHEkBFKRBc9t1aJcf8fRijASdVNmc0+6g2Y678aL4c0iW0GbLOh+0anvp0IpSoyGhwPubJC6oX4EjKRTOGmnpsQAxgRxvAhrQ6PvVYv8Z/xiUIR9ZF54hvlD//FGB9n6qs+DUQGO/2puL/+xlemUyjQ77lTRFSWjE0+BRqThZPg+HD6VDQWoaAISTYXcuxuBAUIiw0Ui/YjjO4VkY9K1AB2TdpyoFl2tpZuHYPP08p/8Z4edvY6CQc7Ka6nO4/NJN9rv73/faxyfLxOtsh31gHZawA3bMTtk56zHBKnbL7th97Ve0ETWjzSdrVFv+2WJ/VbTzCKTlt38=</latexit> 1 む Step 方 : 方 む Step 2 : 1 む Step 0 : 3 む Step 大 大 む 9 8 [Soudry+ む ; active
  64. 1 2 min kwk s.t. 8i : hw, zi i

    w2Rd 2 1 1 2 Lagrangian L := kwk + 2 i=1 <latexit sha1_base64="XcXzG+l8Ay+nJsPqQ6yDt6b7HpQ=">AAACeHicfVDRShtBFJ1stdVU21gf+zIYilYk7EqIvghCfOhLaQSjgWy63J3c3QzOzC4zs9p0yU/4Nb62f+Gv9KmzMUIbSy8MHM49d869J84FN9b3H2rei5XVl6/W1uuvNzbfvG1svbs0WaEZ9lkmMj2IwaDgCvuWW4GDXCPIWOBVfN2t+lc3qA3P1IWd5jiSkCqecAbWUVHjIJRgJ3FCb+kJDU0ho5KfBLOvKhTukzFE/EnwPeJRo+m3/HnR5yBYgCZZVC/aqrXDccYKicoyAcYMAz+3oxK05UzgrB4WBnNg15Di0EEFEs2onJ81ox8cM6ZJpt1Tls7ZPydKkMZMZeyU1Y5muVeR/+zFcsnZJsejkqu8sKjYo3FSCGozWmVGx1wjs2LqADDN3e6UTUADsy7Zej08Q3ecxs/O6EuOGmym98sQdCq5mrlj0/CgQv8TwrcnoUMu52A51efg8rAVdFqd83bztLtIfI28JztkjwTkiJyST6RH+oSRO3JPfpCftV8e9Xa9j49Sr7aY2SZ/lXf4G558wh4=</latexit> w= n X i zi hw, zi i) i (1 <latexit sha1_base64="Yn2t8qmi2c1sfMiD+1bEqImZlBY=">AAACW3icfZDPSsNAEMa38X+stiqevASLUEVKIqIeBT14ERWsFppSJttpXLq7CbsbsYQ+iVd9KA++i9saQas4sPDjm2+ZmS9KOdPG999Kzszs3PzC4pK7XF5ZrVTX1u90kimKTZrwRLUi0MiZxKZhhmMrVQgi4ngfDc7G/ftHVJol8tYMU+wIiCXrMwrGSt1qpR5y6+5Bl4Ux+rvdas1v+JPyfkNQQI0Udd1dKx2GvYRmAqWhHLRuB35qOjkowyjHkRtmGlOgA4ixbVGCQN3JJ5uPvB2r9Lx+ouyTxpuo33/kILQeisg6BZgHPd0bi3/2IjE12fRPOjmTaWZQ0s/B/Yx7JvHGsXg9ppAaPrQAVDG7u0cfQAE1NjzXDc/RHqfw0g66SlGBSdReHoKKBZMje2wc7o/pPyM8fRkt2ZyD6VR/w91BIzhqHN0c1k7PisQXyRbZJnUSkGNySi7INWkSSjLyTF7Ia+ndmXFcp/xpdUrFnw3yo5zNDxWyteg=</latexit> ( i 0) i=1 <latexit sha1_base64="PYfuZ308YuDxv5HJsWTGVXk7/qc=">AAACh3icfZDRShtBFIYnW2s1tRrby94MhkIRSXdFUkEsil70RrTQqOCE5ezk7GZwZnaZmdWmSx7Gp/HWXvo2nU0itLH0wMDHf/7hnPMnhRTWheFjI3ix8HLx1dJy8/XKm9W11vrbc5uXhmOP5zI3lwlYlEJjzwkn8bIwCCqReJFcH9X9ixs0VuT6uxsV2FeQaZEKDs5LcWuPSW8eQCzoFxpSJtKUeklnEilT4IZJSm+3nuhnLJiZNvdpFLfaYSecFH0O0QzaZFZn8Xpjhw1yXirUjkuw9ioKC9evwDjBJY6brLRYAL+GDK88alBo+9XkyjH94JUBTXPjn3Z0ov75owJl7Ugl3lmva+d7tfjPXqLmJrt0t18JXZQONZ8OTktJXU7rCOlAGOROjjwAN8LvTvkQDHDng2422TH64wye+EGnBRpwudmsGJhMCT32x2Zsq6b/GeHHk9GTzzmaT/U5nG93om6n+22nfXA0S3yJvCcb5COJyGdyQL6SM9IjnNyRe/JAfgXLwaegG+xOrUFj9ucd+auCw98r0sWq</latexit> () hw, zi i = 1 i >0 課 <latexit sha1_base64="skSM7VSREQBWmr+GOZy4KcQnWuw=">AAACv3icfVFtaxQxEM6ub9fz7aof/RI8hNbWY1dKFYpQqR/8Ilbw2kJzPWZzs9vQJLsks+q53A/tr9Hs9SrnVRwIPHmemczMk6zSylOSXEbxrdt37t7rrHXvP3j46HFv/cmRL2sncShLXbqTDDxqZXFIijSeVA7BZBqPs4uDVj/+hs6r0n6laYUjA4VVuZJAgRr3vDBA51nOv2/QVropCiTPl7hNvsWFr824Ue/S2ZkVSCD2OJ41r1xQhQZbaPxTUQdu+/ryc6yEm+szsbdEjnv9ZJDMg98E6QL02SIOx+vRjpiUsjZoSWrw/jRNKho14EjJ8HpX1B4rkBdQ4GmAFgz6UTN3Z8ZfBGbC89KFY4nP2eWKBoz3U5OFzHZGv6q15D+1zKx0pvztqFG2qgmtvGqc15pTyVvr+UQ5lKSnAYB0KszO5Tk4kBQ+qNsVHzAs5/BTaPS5QgdUupeNAFcYZWdh2UJst+h/ifDjOjGg4HO66upNcPR6kO4Odr/s9PcPFo532DP2nG2wlL1h++wjO2RDJtkl+xV1orX4fVzENq6uUuNoUfOU/RXx9Dcnv9oV</latexit> w(t + 1) w(t) + n X r(t) → ∞ <latexit sha1_base64="QzvY3r0hmwy5ytRxREh7qUSl4P4=">AAACmnicfZDfShwxFMazY/3TtepqL9uL4FIoIsuMiIogSO1FS2lVcFUwy3Ame2YMJpkhyYjrsG/Vl2lv2wdpZneldZUeCPw433c4OV9SSGFdGP5oBDMvZufmF142F18tLa+0VtfObV4ajl2ey9xcJmBRCo1dJ5zEy8IgqETiRXJzVOsXt2isyPWZGxTYU5BpkQoOzrfi1jdmSxVXYp8yCTqTSJkCd52ktIw3Nh/4PhbMjOUDyjJQCob1gEr64KW/rrjVDjvhqOhTiCbQJpM6iVcb26yf81KhdlyCtVdRWLheBcYJLnHYZKXFAvgNZHjlUYNC26tGhw/pO9/p0zQ3/mlHR91/JypQ1g5U4p31H+20Vjef1RI1tdmle71K6KJ0qPl4cVpK6nJap0r7wiB3cuABuBH+75RfgwHufPbNJvuI/jiDX/2i4wINuNxsVAxMpoQe+mMztlnT/4xw92D05HOOplN9CudbnWins3O63T48miS+QN6QdfKeRGSXHJJP5IR0CSffyU/yi/wO3gYfgs/Bl7E1aExmXpNHFZz9Acluzrk=</latexit> ⌘e r(t)hu(t),zi i zi KKT : u(t) ! 9u⇤ 小 大 <latexit sha1_base64="UY0P+omkPXXD1yHCEdKFFxOnbGg=">AAACbnicfVBdaxNREL1Ztdb0w7SCLyJeGoS0lLBbSutjID74IlYwbaEbwuzNbHrp/VjunS0NS979Nb7qX/Ff+BO8m0bRtHRg4HDmDGfmZIWSnuL4ZyN69PjJytPVZ8219Y3N562t7VNvSydwIKyy7jwDj0oaHJAkheeFQ9CZwrPsql/Pz67ReWnNF5oWONQwMTKXAihQo9ZOqoEus5yXHdrlKdkUb4Kr/0uP9katdtyN58XvgmQB2mxRJ6OtxmE6tqLUaEgo8P4iiQsaVuBICoWzZlp6LEBcwQQvAjSg0Q+r+TMz/jYwY55bF9oQn7P/blSgvZ/qLCjrG/3yrCbvnWV6yZnyd8NKmqIkNOLWOC8VJ8vrpPhYOhSkpgGAcDLczsUlOBAU8mw20/cYnnP4MRh9KtABWbdXpeAmWppZeHaS7tfoISHc/BEGFHJOllO9C04PuslR9+jzYbvXXyS+yl6xHdZhCTtmPfaBnbABE+wr+8a+sx+NX9HL6HX05lYaNRY7L9h/FXV+A1Uwvec=</latexit> ⟨u(t), zi⟩ X i:hu⇤ ,zi i= i=1 0 9 n X <latexit sha1_base64="7xwyPjZ48Y5cbEBNCCinvYAXi48=">AAACs3icfVFdaxQxFM1O/WjXr2197EtwEarWZWapVYRCoT74oFjBbQub7XAnc2cammSGJKNup/sT+wP8Hb4qmJluRbfihcDJOfdybk6SUgrrwvBbJ1i6cfPW7eWV7p279+4/6K2uHdiiMhxHvJCFOUrAohQaR044iUelQVCJxMPkdK/RDz+jsaLQn9y0xImCXItMcHCeins5U+BOOEj6jr7eoSwzwKMhO2/pJKNf2PnxkD6jzFYqrsVONDvWlElvkEIsNiL6vLnpXCL9PbJ5hc5iwUwrPol7/XAQtkWvg2gO+mRe+/FqZ4ulBa8UasclWDuOwtJNajBOcImzLqsslsBPIcexhxoU2kndJjKjjz2T0qww/mhHW/bPiRqUtVOV+M5mV7uoNeQ/tUQtOLvs1aQWuqwcan5pnFWSuoI2cdNUGOROTj0AboTfnfIT8CE7/yndLnuD/nEG33ujDyUacIV5WjMwuRJ65h+bs80G/a8Rvl41euRzjhZTvQ4OhoNoe7D9cau/uzdPfJmsk0dkg0TkJdklb8k+GRFOLsh38oP8DF4E4yAJ0svWoDOfeUj+qkD9AoTs1rM=</latexit> <latexit sha1_base64="pWdS+Qxgl2MvWyQ02yYAPkSMCeE=">AAACx3icfVFNb9QwEHXCR8vytVuOXCxWSAhVUbKqCuJUqRzggCiIbSvV22jinaRWHSeynXaXNCvxMznyT3B2Uz62iJFsPb95oxm/SUopjA3D755/6/aduxub93r3Hzx89Lg/2Do0RaU5jnkhC32cgEEpFI6tsBKPS42QJxKPkvP9Nn90gdqIQn2x8xInOWRKpIKDdVTcr1kuVOxusGdJSi+ZUHT1SOrPzem0YakGHo3Y1W/J1emIMoszWy8WJrDBYuFUhQYpqXhDmQSVSaS/9NvX6GssmO6SGdIo7g/DIFwGvQmiDgxJFwfxwNth04JXOSrLJRhzEoWlndSgreASmx6rDJbAzyHDEwcV5Ggm9dKlhj53zJS6Qd1Rli7ZPytqyI2Z54lTtgOb9VxL/jOX5Gudbfp6UgtVVhYVXzVOK0ltQdsV0KnQyK2cOwBcCzc75WfgbLZuUb0ee4vucxo/uEYfS9RgC/2yZqAzt6rGfTZj2y36nxBm10KHnM/Ruqs3weEoiHaD3U87w739zvFN8pQ8Iy9IRF6RPfKOHJAx4eSHt+H1vYH/3i/8C3+2kvpeV/OE/BX+t5/Nzd7M</latexit> i.e. i zi
  65. ] : w(t) = r(t) · u(t) む <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit>

    <latexit sha1_base64="BTfHnp/5rWUjw9t97GIzTG5w70I=">AAACnnicfZBdaxNBFIYn61eNH0310pvBIKRSw66U6k2hUBEvLK1i2kAnhrOTs9uhM7PLzFlNXPK//CveeKs/w9k0FU3FAwMP73sOZ86bllp5iuNvreja9Rs3b63dbt+5e+/+emfjwbEvKidxIAtduGEKHrWyOCBFGoelQzCpxpP0fL/xTz6h86qwH2hW4shAblWmJFCQxp33b3vCAJ2lGf+8yXe5IJySp5lGLnxlxrXaTeYfrcBp2XsmNNi8cS4ntn7jl7ESbuFujjvduB8vil+FZAldtqyj8UZrW0wKWRm0JDV4f5rEJY1qcKSkxnlbVB5LkOeQ42lACwb9qF4cP+dPgjLhWeHCs8QX6p8TNRjvZyYNnc1f/arXiP/0UrOymbKXo1rZsiK08mJxVmlOBW+S5RPlUJKeBQDpVPg7l2fgQFLIv90WrzAc5/AgLDos0QEV7mktwOVG2Xk4NhdbDf2vEaaXjYFCzslqqlfh+Hk/2envvNvu7u0vE19jj9hj1mMJe8H22Bt2xAZMsq/sO/vBfkY8eh0dRIcXrVFrOfOQ/VXR8Bdfq8/B</latexit> L(w) = i=1 exp( hw, zi i) ; zi := yi xi <latexit sha1_base64="CYBsIAf6ejlzLwUOo9fdk5WPTTo=">AAACaXicfZDfahNBFMZnt2rb+C+pN0UpDAZBpIRdCWkRhEK98EaM0KSFbljOTs4mQ2Zml5nZ0rjkyqfxVp+mz9CXcDbZgKbFAwM/vvMN55wvyQU3NghuPH/rwcNH2zu7jcdPnj573mztDU1WaIYDlolMXyRgUHCFA8utwItcI8hE4HkyO63651eoDc/UmZ3nOJIwUTzlDKyT4uZBJMFOk5R+jzn98JHOY75WrmMeN9tBJ1gWvQthDW1SVz9ued1onLFCorJMgDGXYZDbUQnaciZw0YgKgzmwGUzw0qECiWZULu9Y0DdOGdM00+4pS5fq3z9KkMbMZeKc1Y5ms1eJ9/YSuTHZpsejkqu8sKjYanBaCGozWoVEx1wjs2LuAJjmbnfKpqCBWRdloxF9Qnecxi9u0NccNdhMvysj0BPJ1cIdO4kOK/qfEa7XRkcu53Az1bswfN8Je53et2775LROfIe8Iq/JWxKSI3JCPpM+GRBGfpCf5Bf57d36LX/ff7my+l795wX5p/z2H13vu5w=</latexit> kzi k  1 <latexit sha1_base64="VszzYOV0URl+yCZaxJDyP37+WKk=">AAACYHicfVBNSwMxEE3Xr1q/WvWkl2ARRKTsiqhHQQ9eRAWrglvKbDpbg0l2SbJiXftfvOo/8uovMVsraBUfBF7evGFmXpQKbqzvv5W8sfGJyanydGVmdm5+oVpbvDRJphk2WSISfR2BQcEVNi23Aq9TjSAjgVfR3WFRv7pHbXiiLmwvxZaEruIxZ2Cd1K4uh0+hBHsbxfSxzd1HIA3a1brf8Aegv0kwJHUyxFm7VtoJOwnLJCrLBBhzE/ipbeWgLWcC+5UwM5gCu4Mu3jiqQKJp5YP1+3TdKR0aJ9o9ZelA/d6RgzSmJyPnLFY1o7VC/LMWyZHJNt5v5VylmUXFPgfHmaA2oUU2tMM1Mit6jgDT3O1O2S1oYNYlWKmER+iO03jiBp2mqMEmejMPQXclV313bDfcKth/Rnj4Mjrmcg5GU/1NLrcbwW5j93ynfnA4TLxMVska2SAB2SMH5JickSZh5JE8kxfyWnr3yt6CV/u0eqVhzxL5AW/lAx/HuEo=</latexit> <latexit sha1_base64="bqBQXDCDWA+nPdlWVC4j8BeWJhg=">AAACZHicfZDRShtBFIYna2ttam1UCgVBBkMhKZLuFrHeCIK98MKiglHBDeHs5GwcnJldZs62DUv6NL1t36cv0OdwNkbQKD0w8PGffzjn/EmupKMw/FsL5p49n3+x8LL+avH10pvG8sqZyworsCsyldmLBBwqabBLkhRe5BZBJwrPk+v9qn/+Da2TmTmlUY49DUMjUymAvNRvvDtsxRroKkn59xa123yXH7Wij9TuN5phJ5wUfwzRFJpsWsf95dpWPMhEodGQUODcZRTm1CvBkhQKx/W4cJiDuIYhXno0oNH1yskJY/7eKwOeZtY/Q3yi3v9RgnZupBPvrNZ1s71KfLKX6JnJlO70SmnygtCI28FpoThlvMqHD6RFQWrkAYSVfncursCCIJ9ivR5/QX+cxa9+0FGOFiizH8oY7FBLM/bHDuPNiv5nhB93Rk8+52g21cdw9qkTbXe2T7aae/vTxBfYGttgLRaxz2yPHbBj1mWC/WS/2G/2p/YvWAxWg7e31qA2/bPKHlSwfgOFNLfM</latexit> L(w(t)) = O(1/t) r(t) = ⇥(log t) span({zi }) <latexit sha1_base64="gEzgC3pGdxe+436igLwyn6ffUdY=">AAACanicfVBdSxtBFJ1dbdXtV7RPIsjQtGBLCbsi2kdBH3wRFYwJuCHcndxNhszMLjOzYrrsW3+Nr/pn/A/+CGdjhDYWDwycOfdc7r0nyQU3NgzvPX9h8c3bpeWV4N37Dx8/NVbXLkxWaIZtlolMdxMwKLjCtuVWYDfXCDIR2EnGB3W9c4Xa8Eyd20mOPQlDxVPOwDqp39iMJdiRlqXJQVVbcTn9Jyn93edx9b3faIatcAr6kkQz0iQznPZXvZ14kLFCorJMgDGXUZjbXgnaciawCuLCYA5sDEO8dFSBRNMrp4dU9JtTBjTNtHvK0qn6d0cJ0piJTJyzXtPM12rxv7VEzk226a9eyVVeWFTsaXBaCGozWqdEB1wjs2LiCDDN3e6UjUADsy7LIIgP0R2n8dgNOslRg830jzIGPZRcVe7YYfyzZq8Z4frZ6JjLOZpP9SW52G5Fu63ds53m/sEs8WWyQb6QLRKRPbJPjsgpaRNG/pAbckvuvAd/zV/3N56svjfr+Uz+gf/1EVTtvRA=</latexit> 8 : Pn <latexit sha1_base64="CfNqES2E21yVkH2Ja8Pg7riXN/I=">AAACYXicfZBdSxtBFIYna1tj+pUoXnkzNBRiKWG3iHojCPbCG6kFo4IbwtnJyWbIfCwzZ0vDkh/jrf4ir/tHnMQU2lh6YODhPe9wznmzQklPcfxQi9ZevHy1Xt9ovH7z9t37Zmvz0tvSCewJq6y7zsCjkgZ7JEnhdeEQdKbwKpuczPtXP9B5ac0FTQvsa8iNHEkBFKRBc9t1aJcf8fRijASdVNmc0+6g2Y678aL4c0iW0GbLOh+0anvp0IpSoyGhwPubJC6oX4EjKRTOGmnpsQAxgRxvAhrQ6PvVYv8Z/xiUIR9ZF54hvlD//FGB9n6qs+DUQGO/2puL/+xlemUyjQ77lTRFSWjE0+BRqThZPg+HD6VDQWoaAISTYXcuxuBAUIiw0Ui/YjjO4VkY9K1AB2TdpyoFl2tpZuHYPP08p/8Z4edvY6CQc7Ka6nO4/NJN9rv73/faxyfLxOtsh31gHZawA3bMTtk56zHBKnbL7th97Ve0ETWjzSdrVFv+2WJ/VbTzCKTlt38=</latexit> 1 む Step 方 : 方 む Step 2 : 1 む Step 0 : 3 む Step 大 大 む 1 9 [Soudry+ む ; active
  66. u(t) ! u⇤ <latexit sha1_base64="jx56rdHtP6hdTncqdZyKUtPpE9g=">AAACaHicfVBNSxxBEO2d+JX1I2s8hKCHxkVQCcuMiMlRMAcvQQOuCs6y1PTWrI3dPUN3Tcgy7MVf41X/jX8hv8KedRWzigUFj1evqHovyZV0FIb3teDD1PTM7NzH+vzC4tKnxvLnU5cVVmBbZCqz5wk4VNJgmyQpPM8tgk4UniVXB9X87A9aJzNzQoMcOxr6RqZSAHmq21iLNdBlkvJik7Z4TBl/Jrrb3UYzbIWj4q9BNAZNNq7j7nJtN+5lotBoSChw7iIKc+qUYEkKhcN6XDjMQVxBHy88NKDRdcqRjSHf8EyPp5n1bYiP2JcbJWjnBjrxyupHNzmryDdniZ64TOmPTilNXhAa8Xg4LRT37quMeE9aFKQGHoCw0v/OxSVYEOSTrNfjn+jNWfzlDx3laIEyu13GYPtamqE324+/Veg9Ifx9Enrkc44mU30NTnda0V5r7/duc/9gnPgcW2XrbJNF7DvbZ4fsmLWZYNfsht2yu9q/oBF8Cb4+SoPaeGeF/VfB+gO5SbrP</latexit> (Step ) (Step ) <latexit

    sha1_base64="kwSS7tPftupXdT5JReJqK7rV84M=">AAACgXicfVDbahRBEO2deInrbaOPeWlchCTKMhNiFEQIxAdfxAhuEkgvS01vzaRJX4buGs0y7Jf4NXlNvsC/sWezAd2IBU2fOnWKqjp5pVWgNP3VSVbu3L13f/VB9+Gjx0+e9taeHQZXe4lD6bTzxzkE1MrikBRpPK48gsk1HuVn+2396Dv6oJz9RtMKRwZKqwolgSI17r0RBug0L/iPDdrkH/hNWo+3uNCu5MRfcZE7PQlTEz9xrqJw3Oung3Qe/DbIFqDPFnEwXuvsiImTtUFLUkMIJ1la0agBT0pqnHVFHbACeQYlnkRowWAYNfP7ZvxlZCa8cD4+S3zO/tnRgAntelHZrh+Way35z1puliZT8W7UKFvVhFZeDy5qzcnx1jw+UR4l6WkEIL2Ku3N5Ch4kRYu7XfER43EeP8dBXyr0QM5vNQJ8aZSdxWNL8bpF/xPC+Y0wouhztuzqbXC4Pch2B7tfd/p7+wvHV9k6e8E2WMbesj32iR2wIZPsJ7tgl+wqWUk2kzTZvpYmnUXPc/ZXJO9/A1v6wxw=</latexit> w(t) = u⇤ log t + ⇠(t) k⇠(t)k = o(log t) <latexit sha1_base64="g1WUM3ssA9u/vs2Cg42oDbMjQ0Y=">AAACa3icfVBdaxNBFJ2sX238Su1bFRwMhVQk7EppfREK9cEXsYJpC50Q7k5utkPnY5m5Wxq2efTX+Ko/pj+i/8HZNIKm4oVhDueey7n35KVWgdL0qpXcuXvv/oOV1fbDR4+fPO2sPTsMrvISB9Jp549zCKiVxQEp0nhcegSTazzKz/ab/tE5+qCc/UrTEocGCqsmSgJFatR5KS5F7vQ4TE38xIXq0Za4fO96QruC09ao00376bz4bZAtQJct6mC01toWYycrg5akhhBOsrSkYQ2elNQ4a4sqYAnyDAo8idCCwTCs55fM+GZkxnzifHyW+Jz9c6IGE5pVo9IAnYblXkP+s5ebJWeavBvWypYVoZU3xpNKc3K8iYmPlUdJehoBSK/i7lyeggdJMcx2W3zAeJzHT9Hoc4keyPnXtQBfGGVn8dhCvGnQ/4Rw8VsYUcw5W071Njh82892+jtftrt7+4vEV9hz9or1WMZ22R77yA7YgEn2jX1nP9jP1nWynmwkL26kSWsxs87+qmTzF/AIvM4=</latexit> 1 1 <latexit sha1_base64="JYVCGexSDsnZKfXBe9nvxyJ/BTE=">AAAEsXiclVNdb9MwFE1DgVG+VnjkxaICtds6mmwavCBNGg+8oA2JbkNzWxzHyaw6TnAc2OTmF/IL+Bm8wgs3abZ+bALNUpSb63PPuec69hLBU93r/azZt+q379xdude4/+Dho8erzSeHaZwpyvo0FrE69kjKBJesr7kW7DhRjESeYEfeeK/YP/rGVMpj+UmfJ2wQkVDygFOiITVq2j5C2GMhl4YIHkrm5w0EC0dEn3oBytBbhANFqLnIfM8Nnlx+4MkU/3IZlo3W0DpQx8JPzyN44TOOqR/rNhZxiHRnaLpODlzpV6WNA1gXCyJDweYp8g2zSJFjVYKuUhVqkyW9ydCdw7h5niOML/sVLACKmdh65WBR0EwJQJiHp7pTVp04qHvh9wZNX3IVvZbVWzcpH7rd6xzmxi1ohy7wdtF+Zaukd6aKw61Z++VrMD+GhQM7GC20MsQJU8n/TrGQbTuvqi46G0CNmfRnf9RotdXb7JULXQ2cKmhZ1ToYNWvb2I9pFjGpqSBpeuL0Ej0wRGlOYZANnKUsIXRMQnYCoSQRSwemvA85egEZHwWxgkdqVGbnKwyJ0sIOIAuv6fJekbx2z4uWlHXwZmC4TDLNJJ0KB5lAOkbFZUM+V4xqcQ4BoYpD74ieEjgZDVey0cDvGJhT7AMI7cOgiY7VmsFEhRGXOZgN8UYR/QtIzi6AEMGcneWpXg0O3U1nZ3Pn43Zrd6+a+Ir1zHputS3Hem3tWu+tA6tvUfuH/cv+bf+pb9U/17/UvSnUrlU1T62FVR//BesCjnE=</latexit> w u⇤ + ⇠ · (log t) u= w = p u⇤ + ⇠ · (log t) u = kwk = p1 + 2hu⇤ , ⇠i · (log t) 1 + k⇠k2 (log t) 2 kwk ✓ 1 + 2hu⇤ ,◆ ⇠i· (log t) 1 + k⇠k2 (log t)2 2 2 ✓ ⇠ ◆ hu⇤ , ⇠i 3hu⇤ , ⇠i2 k⇠k2 = u⇤ + ⇠ 1 hu⇤ , ⇠i + 3hu⇤ , ⇠i 2 k⇠k log t + = u⇤ + log t 1 2 log2 t log t log t 2 log t 2 ? 1 = u⇤ + Pu?⇤ ⇠ · (log t) 1 O(1/ log2 t), = u⇤ + Pu⇤ ⇠ · (log t) O(1/ log t), <latexit sha1_base64="JYVCGexSDsnZKfXBe9nvxyJ/BTE=">AAAEsXiclVNdb9MwFE1DgVG+VnjkxaICtds6mmwavCBNGg+8oA2JbkNzWxzHyaw6TnAc2OTmF/IL+Bm8wgs3abZ+bALNUpSb63PPuec69hLBU93r/azZt+q379xdude4/+Dho8erzSeHaZwpyvo0FrE69kjKBJesr7kW7DhRjESeYEfeeK/YP/rGVMpj+UmfJ2wQkVDygFOiITVq2j5C2GMhl4YIHkrm5w0EC0dEn3oBytBbhANFqLnIfM8Nnlx+4MkU/3IZlo3W0DpQx8JPzyN44TOOqR/rNhZxiHRnaLpODlzpV6WNA1gXCyJDweYp8g2zSJFjVYKuUhVqkyW9ydCdw7h5niOML/sVLACKmdh65WBR0EwJQJiHp7pTVp04qHvh9wZNX3IVvZbVWzcpH7rd6xzmxi1ohy7wdtF+Zaukd6aKw61Z++VrMD+GhQM7GC20MsQJU8n/TrGQbTuvqi46G0CNmfRnf9RotdXb7JULXQ2cKmhZ1ToYNWvb2I9pFjGpqSBpeuL0Ej0wRGlOYZANnKUsIXRMQnYCoSQRSwemvA85egEZHwWxgkdqVGbnKwyJ0sIOIAuv6fJekbx2z4uWlHXwZmC4TDLNJJ0KB5lAOkbFZUM+V4xqcQ4BoYpD74ieEjgZDVey0cDvGJhT7AMI7cOgiY7VmsFEhRGXOZgN8UYR/QtIzi6AEMGcneWpXg0O3U1nZ3Pn43Zrd6+a+Ir1zHputS3Hem3tWu+tA6tvUfuH/cv+bf+pb9U/17/UvSnUrlU1T62FVR//BesCjnE=</latexit> O O ✓ ✓ <latexit sha1_base64="Z6COF0y72RhvKZPXj3w1+T2BmN4=">AAACg3icfZDfShtBFMYn2/ovrRrby94MDQXbaNiJQb0RBHvRm6KFRgU3htnJ2Tg4M7vOnC0bln2UPk1v6wP0bTr5U9AoPTDw4/vO4cz54kxJh2H4pxa8eLm0vLK6Vn/1en1js7H15tyluRXQE6lK7WXMHShpoIcSFVxmFriOFVzEtycT/+IHWCdT8x3HGfQ1HxmZSMHRS4PGQYSJ5aJkVRm5O4slaxVVRY8oo7szi3WK1oz2DovrTut0u7je+zhoNMN2OC36FNgcmmReZ4OtWjcapiLXYFAo7twVCzPsl9yiFAqqepQ7yLi45SO48mi4BtcvpxdW9INXhjRJrX8G6VR9OFFy7dxYx75Tc7xxi95EfNaL9cJmTA77pTRZjmDEbHGSK4opncRHh9KCQDX2wIWV/u9U3HAfDvqQ6/XoM/jjLHz1i04zsBxT+6mMuB1paSp/7CjamdD/Gnnxr9GTz5ktpvoUzjtttt/e/9ZtHp/ME18l78h7sk0YOSDH5As5Iz0iyE/yi/wm98FS0Ao6QXfWGtTmM2/JowqO/gJYGcPq</latexit> 2 1 hu, u⇤ i = 1 O ✓ 1 2 log t 方 <latexit sha1_base64="TPhVKMoSREKLrOoxvsxEq70IYXU=">AAACl3icfVBdaxNBFJ2sXzV+NNUn8WUwCLXUsFtK9UUsVKQg0hZMW+jEcHdydzN0ZnaZuSuGZX+Uv0Z803/ibBJFU/HCwOGcc7lzTlpq5SmOv3Wia9dv3Ly1drt75+69++u9jQenvqicxKEsdOHOU/ColcUhKdJ4XjoEk2o8Sy8PWv3sEzqvCvuBZiWODORWZUoCBWrceyc02FwjFwZomma82v6NxlvCLcRXPOHP+ZHQmNGmyBzIOmlqoYv84w6nRjiVT+nZuNePB/F8+FWQLEGfLed4vNHZFZNCVgYtSQ3eXyRxSaMaHCmpsemKymMJ8hJyvAjQgkE/quepG/40MBOeFS48S3zO/rlRg/F+ZtLgbBP5Va0l/6mlZuUyZS9HtbJlRWjl4nBWaU4FbyvlE+VQkp4FANKp8HcupxBKolB8tyveYAjn8H04dFSiAyrcVi3A5UbZJoTNxXaL/meEz7+MAYWek9VWr4LTnUGyN9g72e3vHywbX2OP2RO2yRL2gu2zQ3bMhkyyL+wr+85+RI+i19Hb6HBhjTrLnYfsr4lOfgKxcsxp</latexit> 2 9 r(t) = ⇥(log t) & <latexit sha1_base64="CfNqES2E21yVkH2Ja8Pg7riXN/I=">AAACYXicfZBdSxtBFIYna1tj+pUoXnkzNBRiKWG3iHojCPbCG6kFo4IbwtnJyWbIfCwzZ0vDkh/jrf4ir/tHnMQU2lh6YODhPe9wznmzQklPcfxQi9ZevHy1Xt9ovH7z9t37Zmvz0tvSCewJq6y7zsCjkgZ7JEnhdeEQdKbwKpuczPtXP9B5ac0FTQvsa8iNHEkBFKRBc9t1aJcf8fRijASdVNmc0+6g2Y678aL4c0iW0GbLOh+0anvp0IpSoyGhwPubJC6oX4EjKRTOGmnpsQAxgRxvAhrQ6PvVYv8Z/xiUIR9ZF54hvlD//FGB9n6qs+DUQGO/2puL/+xlemUyjQ77lTRFSWjE0+BRqThZPg+HD6VDQWoaAISTYXcuxuBAUIiw0Ui/YjjO4VkY9K1AB2TdpyoFl2tpZuHYPP08p/8Z4edvY6CQc7Ka6nO4/NJN9rv73/faxyfLxOtsh31gHZawA3bMTtk56zHBKnbL7th97Ve0ETWjzSdrVFv+2WJ/VbTzCKTlt38=</latexit> ◆ 1 13 log3 t log t ◆ ◆ p1 = 1 1+x 3 1 3 2 2 x + 8 x + O(x )
  67. 2 O(1/ log t) <latexit sha1_base64="uunEm6Q7k5mbcYgF7u+1vAeepEU=">AAACUHicfZDPShxBEMZrVmN0kvj36GVwCRiRzYyI8SjoIRdRwV0FZ5Ga3tq1tbtn6K4Rl2HfwWvyWN58k9y0d13BrCEFTf/46muq+ssKJR3H8WNQm5r+MPNxdi789PnL/MLi0nLL5aUV1BS5yu15ho6UNNRkyYrOC0uoM0Vn2c3+sH92S9bJ3Jxyv6C2xp6RXSmQvdQ6Wk++87fLxXrciEcVvYdkDHUY1/HlUrCddnJRajIsFDp3kcQFtyu0LIWiQZiWjgoUN9ijC48GNbl2NVp3EH31Sifq5tYfw9FIffuiQu1cX2feqZGv3GRvKP6zl+mJydzdbVfSFCWTES+Du6WKOI+GWUQdaUmw6ntAYaXfPRJXaFGwTywM0wPyn7N06AcdFWSRc7tRpWh7WpqB/2wv3RzS/4x492r05HNOJlN9D62tRrLT2DnZru/tjxOfhVVYg3VI4AfswU84hiYIuIZ7+AW/g4fgT/BUC16srzeswF9VC58BUFCzoQ==</latexit> O(1/t) : む [Soudry+

    ] 非 方 8 1 方 方 方 大 Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. 3 9 📄 : む <latexit sha1_base64="hSV+wnmgdmUZ/yrJb3dbR+ORChM=">AAACVnicfZDdSiNBEIV7xv+sulEvvRkMC7oscUbE9VLQC29EBaOCk5WaTmVs7J+hu0YMQ17D293H0pcROzGCRtmCho9Tp6mqkxVSOIrjpyCcmJyanpmdq32bX1j8Xl9aPnemtBxb3EhjLzNwKIXGFgmSeFlYBJVJvMhu9wf9izu0Thh9Rr0C2wpyLbqCA3kpPV5PNlNp8j9btHFdb8TNeFjRZ0hG0GCjOrleCrbTjuGlQk1cgnNXSVxQuwJLgkvs19LSYQH8FnK88qhBoWtXw6X70Q+vdKKusf5piobq+x8VKOd6KvNOBXTjxnsD8ctepsYmU3e3XQldlISavw7uljIiEw0SiTrCIifZ8wDcCr97xG/AAiefW62WHqA/zuKRH3RcoAUy9meVgs2V0H1/bJ7+GtD/jHD/ZvTkc07GU/0M51vNZKe5c7rd2NsfJT7LVtkaW2cJ+8322CE7YS3GWcEe2F/2L3gMnsOpcObVGgajPyvsQ4X1F0KgtQw=</latexit>
  68. <latexit sha1_base64="ra3lw4QEFs5xkMELv+6F8OkhKs0=">AAACwXicfZFda9swFIZldx+du4+0u9yNWBh0oQv2KN0oFArZxW7GOljSQhWMrBw7orJspONtQfUPHfszk9MMlmTsgODlPY90pFdZraTFOP4ZhDv37j94uPso2nv85Omz3v7BxFaNETAWlarMVcYtKKlhjBIVXNUGeJkpuMxuRl3/8hsYKyv9FRc1TEteaJlLwdFbaa+ZHOJrenpGE/qGMgU5MsV1oYCy3HDhWMlxnuXue+u51rHbNYPdtkdbYDpY59KBxygzspgjM8vD014/HsbLotsiWYk+WdVFuh8cs1klmhI0CsWtvU7iGqeOG5RCQRuxxkLNxQ0v4NpLzUuwU7fMp6WvvDOjeWX80kiX7t87HC+tXZSZJ7tb281eZ/6zl5UbkzF/P3VS1w2CFneD80ZRrGgXPp1JAwLVwgsujPR3p2LOfXzovyiK2AfwjzPwyQ/6XIPhWJmBY9wUpdStf2zBjjr1P5D/+AN65XNONlPdFpO3w+RkePLluH8+WiW+S16Ql+SQJOQdOScfyQUZE0F+BUEQBXvhKJRhHZo7NAxWe56TtQrdb3iB3Js=</latexit> V (t) := 1 <latexit sha1_base64="pT9TLwiruJTBXgk2/L6qHWToFHY=">AAACY3icfZDfShtBFMYna7U2/ou20AsRlgZBRcNuEfVGCNgLb8QUmhjIhnB2cpIMzswuM2clYcnT9FYfyAfoe3TyR6hRemCGH9/5hjPni1MpLAXBc8Fb+rC88nH1U3FtfWNzq7S907BJZjjWeSIT04zBohQa6yRIYjM1CCqWeBffX036dw9orEj0Lxql2FbQ16InOJCTOqWvdHl7EOEwnV0nURclweFhp1QOKsG0/LcQzqHM5lXrbBdOo27CM4WauARrW2GQUjsHQ4JLHBejzGIK/B762HKoQaFt59MNxv6+U7p+LzHuaPKn6r8vclDWjlTsnApoYBd7E/HdXqwWJlPvop0LnWaEms8G9zLpU+JP4vG7wiAnOXIA3Aj3d58PwAAnF2KxGP1At5zBGzfoNkUDlJijPALTV0KP3bL96HhC/zPC8MXoyOUcLqb6FhrfK+FZ5eznabl6NU98le2yb+yAheycVdk1q7E642zMfrNH9lT44615O96XmdUrzN98Zq/K2/sLNlm4uQ==</latexit> <latexit sha1_base64="LQjZYcsM1FBT7BVfqYAlPF9Sn5c=">AAACXXicfZBNSwMxEIbT9bt+tXrw4GWxCCJSdkXUkwh68CIqWBXcUmbTaQ1Nsksyq5alP8Wr/iZP/hXTD0GrOBB4eOcNM/PGqRSWguC94E1MTk3PzM4V5xcWl5ZL5ZUbm2SGY40nMjF3MViUQmONBEm8Sw2CiiXexp2Tfv/2EY0Vib6mbop1BW0tWoIDOalRKkdNlARHUQxmiI1SJagGg/J/QziCChvVZaNc2IuaCc8UauISrL0Pg5TqORgSXGKvGGUWU+AdaOO9Qw0KbT0f7N7zN53S9FuJcU+TP1C//8hBWdtVsXMqoAc73uuLf/ZiNTaZWof1XOg0I9R8OLiVSZ8Svx+M3xQGOcmuA+BGuN19/gAGOLn4isXoFN1xBs/doIsUDVBitvMITFsJ3XPHtqOdPv1nhOcvoyOXczie6m+42a2G+9X9q73K8cko8Vm2zjbYFgvZATtmZ+yS1RhnT+yFvbK3woc35S14S0OrVxj9WWU/ylv7BCXjt2k=</latexit>

    >¯ V (t)  <latexit sha1_base64="p0B7cx5cj5qkrIjmeYG55Kv3oy8=">AAACW3icfZDPattAEMZXcv+4alrbCT31stQU0lKMFIybYyA55BLqQm0HImFG67GzZHcldkelRvhJck0fqoe+S9eOC61TOrDw45tvmZkvL5V0FMc/grDx6PGTp81n0fO9Fy9b7c7+2BWVFTgShSrsZQ4OlTQ4IkkKL0uLoHOFk/zmdN2ffEXrZGG+0LLETMPCyLkUQF6atlvjQ3rHU4U8naEimLa7cS/eFH8IyRa6bFvDaSfop7NCVBoNCQXOXSVxSVkNlqRQuIrSymEJ4gYWeOXRgEaX1ZvNV/ytV2Z8Xlj/DPGN+uePGrRzS517pwa6dru9tfjPXq53JtP8OKulKStCI+4HzyvFqeDrWPhMWhSklh5AWOl35+IaLAjy4UVReob+OIsXftCnEi1QYd/XKdiFlmblj12kH9b0PyN8+2305HNOdlN9COOjXjLoDT73uyen28Sb7DV7ww5Zwj6yE3bOhmzEBKvYLbtj34OfYSOMwr17axhs/xywvyp89QuVkLWo</latexit> t = O(exp(exp( )) [Bao ŵ <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> 方 非 比 方 6 2 Bao, H (2026). Non-asymptotic implicit bias of logistic regression at early-stage gradient descent dynamics. 5 9 📄 ¯>0 ⌧ w(t) w⇤ , kw(t)k kw⇤ k <latexit sha1_base64="VOobOnK2yFg3/q+MVGdJtHfHXog=">AAACVnicfZDdSgMxEIWz63/9q3rpzWIRRKTsiqhXUqgX3ogKVgW3yGw6raFJdklmxbL0NbzVx9KXEdNaQas4EPg4c8LMnCSTwlIYvnn+xOTU9MzsXGl+YXFpubyyemXT3HBs8FSm5iYBi1JobJAgiTeZQVCJxOukWx/0rx/QWJHqS+pl2FTQ0aItOJCT4jgBE7dQEhyFd+VKWA2HFfyGaAQVNqrzuxVvL26lPFeoiUuw9jYKM2oWYEhwif1SnFvMgHehg7cONSi0zWK4dD/YdEoraKfGPU3BUP3+owBlbU8lzqmA7u14byD+2UvU2GRqHzYLobOcUPPPwe1cBpQGg0SCljDISfYcADfC7R7wezDAyeVWKsXH6I4zeOoGnWVogFKzXcRgOkrovju2E+8M6D8jPH4ZHbmco/FUf8PVbjXar+5f7FVq9VHis2ydbbAtFrEDVmMn7Jw1GGcZe2LP7MV79d79KX/m0+p7oz9r7Ef55Q+8W7XP</latexit> ]
  69. <latexit sha1_base64="ra3lw4QEFs5xkMELv+6F8OkhKs0=">AAACwXicfZFda9swFIZldx+du4+0u9yNWBh0oQv2KN0oFArZxW7GOljSQhWMrBw7orJspONtQfUPHfszk9MMlmTsgODlPY90pFdZraTFOP4ZhDv37j94uPso2nv85Omz3v7BxFaNETAWlarMVcYtKKlhjBIVXNUGeJkpuMxuRl3/8hsYKyv9FRc1TEteaJlLwdFbaa+ZHOJrenpGE/qGMgU5MsV1oYCy3HDhWMlxnuXue+u51rHbNYPdtkdbYDpY59KBxygzspgjM8vD014/HsbLotsiWYk+WdVFuh8cs1klmhI0CsWtvU7iGqeOG5RCQRuxxkLNxQ0v4NpLzUuwU7fMp6WvvDOjeWX80kiX7t87HC+tXZSZJ7tb281eZ/6zl5UbkzF/P3VS1w2CFneD80ZRrGgXPp1JAwLVwgsujPR3p2LOfXzovyiK2AfwjzPwyQ/6XIPhWJmBY9wUpdStf2zBjjr1P5D/+AN65XNONlPdFpO3w+RkePLluH8+WiW+S16Ql+SQJOQdOScfyQUZE0F+BUEQBXvhKJRhHZo7NAxWe56TtQrdb3iB3Js=</latexit> V (t) := 1 <latexit sha1_base64="pT9TLwiruJTBXgk2/L6qHWToFHY=">AAACY3icfZDfShtBFMYna7U2/ou20AsRlgZBRcNuEfVGCNgLb8QUmhjIhnB2cpIMzswuM2clYcnT9FYfyAfoe3TyR6hRemCGH9/5hjPni1MpLAXBc8Fb+rC88nH1U3FtfWNzq7S907BJZjjWeSIT04zBohQa6yRIYjM1CCqWeBffX036dw9orEj0Lxql2FbQ16InOJCTOqWvdHl7EOEwnV0nURclweFhp1QOKsG0/LcQzqHM5lXrbBdOo27CM4WauARrW2GQUjsHQ4JLHBejzGIK/B762HKoQaFt59MNxv6+U7p+LzHuaPKn6r8vclDWjlTsnApoYBd7E/HdXqwWJlPvop0LnWaEms8G9zLpU+JP4vG7wiAnOXIA3Aj3d58PwAAnF2KxGP1At5zBGzfoNkUDlJijPALTV0KP3bL96HhC/zPC8MXoyOUcLqb6FhrfK+FZ5eznabl6NU98le2yb+yAheycVdk1q7E642zMfrNH9lT44615O96XmdUrzN98Zq/K2/sLNlm4uQ==</latexit> <latexit sha1_base64="LQjZYcsM1FBT7BVfqYAlPF9Sn5c=">AAACXXicfZBNSwMxEIbT9bt+tXrw4GWxCCJSdkXUkwh68CIqWBXcUmbTaQ1Nsksyq5alP8Wr/iZP/hXTD0GrOBB4eOcNM/PGqRSWguC94E1MTk3PzM4V5xcWl5ZL5ZUbm2SGY40nMjF3MViUQmONBEm8Sw2CiiXexp2Tfv/2EY0Vib6mbop1BW0tWoIDOalRKkdNlARHUQxmiI1SJagGg/J/QziCChvVZaNc2IuaCc8UauISrL0Pg5TqORgSXGKvGGUWU+AdaOO9Qw0KbT0f7N7zN53S9FuJcU+TP1C//8hBWdtVsXMqoAc73uuLf/ZiNTaZWof1XOg0I9R8OLiVSZ8Svx+M3xQGOcmuA+BGuN19/gAGOLn4isXoFN1xBs/doIsUDVBitvMITFsJ3XPHtqOdPv1nhOcvoyOXczie6m+42a2G+9X9q73K8cko8Vm2zjbYFgvZATtmZ+yS1RhnT+yFvbK3woc35S14S0OrVxj9WWU/ylv7BCXjt2k=</latexit>

    >¯ V (t)  <latexit sha1_base64="p0B7cx5cj5qkrIjmeYG55Kv3oy8=">AAACW3icfZDPattAEMZXcv+4alrbCT31stQU0lKMFIybYyA55BLqQm0HImFG67GzZHcldkelRvhJck0fqoe+S9eOC61TOrDw45tvmZkvL5V0FMc/grDx6PGTp81n0fO9Fy9b7c7+2BWVFTgShSrsZQ4OlTQ4IkkKL0uLoHOFk/zmdN2ffEXrZGG+0LLETMPCyLkUQF6atlvjQ3rHU4U8naEimLa7cS/eFH8IyRa6bFvDaSfop7NCVBoNCQXOXSVxSVkNlqRQuIrSymEJ4gYWeOXRgEaX1ZvNV/ytV2Z8Xlj/DPGN+uePGrRzS517pwa6dru9tfjPXq53JtP8OKulKStCI+4HzyvFqeDrWPhMWhSklh5AWOl35+IaLAjy4UVReob+OIsXftCnEi1QYd/XKdiFlmblj12kH9b0PyN8+2305HNOdlN9COOjXjLoDT73uyen28Sb7DV7ww5Zwj6yE3bOhmzEBKvYLbtj34OfYSOMwr17axhs/xywvyp89QuVkLWo</latexit> t = O(exp(exp( )) V (t) ⇡ [Bao p fa st <latexit sha1_base64="Z33XfrCdnuv0JpaVBFn5pCzzQh4=">AAACZHicfZDfSltBEMY3x9ZqqjYqhUJBlgbBFgnnFLFeCvaiN6UWTBQ8IczZTI6L+6+7c4rhEJ+mt+379AV8jm5iCm0sHVj48c03zM5XOCUDpenPRrL06PHyk5XV5tO19Y1nrc2tXrCVF9gVVll/UUBAJQ12SZLCC+cRdKHwvLg+mfbPv6IP0pozGjvsayiNHEkBFKVB60Vvj17zHJzz9obn4YunvAStYdBqp510VvwhZHNos3mdDjYbB/nQikqjIaEghMssddSvwZMUCifNvAroQFxDiZcRDWgM/Xp2woTvRmXIR9bHZ4jP1D8natAhjHURnRroKiz2puI/e4Ve2Eyjo34tjasIjbhfPKoUJ8un+fCh9ChIjSOA8DL+nYsr8CAopths5u8xHufxY1z0yaEHsv5NnYMvtTSTeGyZ70/pf0a4+W2MFHPOFlN9CL23neywc/j5oH18Mk98hb1kr9gey9g7dsw+sFPWZYLdsm/sO/vRuEvWku3k+b01acxnttlflez8ApWRueo=</latexit> <latexit sha1_base64="JfMY6Mvt1u3czNJA7AWwaXD/x1E=">AAACcXicfVDRShtBFJ2sVdNobWKfii+DQdAqYbeI+ijoQ19EC0YFJ4S7k7txcGZ2mZlVw7pf0K/xVb+k39Ef6GyMoLH0wMCZc8/l3nviTArrwvB3LZj5MDs3X//YWFj8tPS52Vo+s2luOHZ5KlNzEYNFKTR2nXASLzKDoGKJ5/H1QVU/v0FjRapP3SjDnoKhFong4LzUb66xxAAvmAJ3FSf0dj3cKAt2//rP7st+sx12wjHoexJNSJtMcNJv1bbZIOW5Qu24BGsvozBzvQKME1xi2WC5xQz4NQzx0lMNCm2vGN9T0jWvDGiSGv+0o2P1dUcBytqRir2z2tNO1yrxn7VYTU12yV6vEDrLHWr+PDjJJXUprcKiA2GQOznyBLgRfnfKr8AH5nykjQY7RH+cwSM/6DhDAy413woGZqiELv2xQ7ZVsf8Z4e7F6JnPOZpO9T05+96Jdjo7P7fb+weTxOtkhaySdRKRXbJPfpAT0iWc/CIP5JE81f4EXwMarD5bg9qk5wt5g2DzL8NUvvo=</latexit> w(0) kw(0)k 方 非 比 方 6 2 Bao, H (2026). Non-asymptotic implicit bias of logistic regression at early-stage gradient descent dynamics. 6 9 📄 ¯>0 ⌧ w(t) w⇤ , kw(t)k kw⇤ k <latexit sha1_base64="VOobOnK2yFg3/q+MVGdJtHfHXog=">AAACVnicfZDdSgMxEIWz63/9q3rpzWIRRKTsiqhXUqgX3ogKVgW3yGw6raFJdklmxbL0NbzVx9KXEdNaQas4EPg4c8LMnCSTwlIYvnn+xOTU9MzsXGl+YXFpubyyemXT3HBs8FSm5iYBi1JobJAgiTeZQVCJxOukWx/0rx/QWJHqS+pl2FTQ0aItOJCT4jgBE7dQEhyFd+VKWA2HFfyGaAQVNqrzuxVvL26lPFeoiUuw9jYKM2oWYEhwif1SnFvMgHehg7cONSi0zWK4dD/YdEoraKfGPU3BUP3+owBlbU8lzqmA7u14byD+2UvU2GRqHzYLobOcUPPPwe1cBpQGg0SCljDISfYcADfC7R7wezDAyeVWKsXH6I4zeOoGnWVogFKzXcRgOkrovju2E+8M6D8jPH4ZHbmco/FUf8PVbjXar+5f7FVq9VHis2ydbbAtFrEDVmMn7Jw1GGcZe2LP7MV79d79KX/m0+p7oz9r7Ef55Q+8W7XP</latexit> ]
  70. <latexit sha1_base64="ra3lw4QEFs5xkMELv+6F8OkhKs0=">AAACwXicfZFda9swFIZldx+du4+0u9yNWBh0oQv2KN0oFArZxW7GOljSQhWMrBw7orJspONtQfUPHfszk9MMlmTsgODlPY90pFdZraTFOP4ZhDv37j94uPso2nv85Omz3v7BxFaNETAWlarMVcYtKKlhjBIVXNUGeJkpuMxuRl3/8hsYKyv9FRc1TEteaJlLwdFbaa+ZHOJrenpGE/qGMgU5MsV1oYCy3HDhWMlxnuXue+u51rHbNYPdtkdbYDpY59KBxygzspgjM8vD014/HsbLotsiWYk+WdVFuh8cs1klmhI0CsWtvU7iGqeOG5RCQRuxxkLNxQ0v4NpLzUuwU7fMp6WvvDOjeWX80kiX7t87HC+tXZSZJ7tb281eZ/6zl5UbkzF/P3VS1w2CFneD80ZRrGgXPp1JAwLVwgsujPR3p2LOfXzovyiK2AfwjzPwyQ/6XIPhWJmBY9wUpdStf2zBjjr1P5D/+AN65XNONlPdFpO3w+RkePLluH8+WiW+S16Ql+SQJOQdOScfyQUZE0F+BUEQBXvhKJRhHZo7NAxWe56TtQrdb3iB3Js=</latexit> V (t) := 1 <latexit sha1_base64="pT9TLwiruJTBXgk2/L6qHWToFHY=">AAACY3icfZDfShtBFMYna7U2/ou20AsRlgZBRcNuEfVGCNgLb8QUmhjIhnB2cpIMzswuM2clYcnT9FYfyAfoe3TyR6hRemCGH9/5hjPni1MpLAXBc8Fb+rC88nH1U3FtfWNzq7S907BJZjjWeSIT04zBohQa6yRIYjM1CCqWeBffX036dw9orEj0Lxql2FbQ16InOJCTOqWvdHl7EOEwnV0nURclweFhp1QOKsG0/LcQzqHM5lXrbBdOo27CM4WauARrW2GQUjsHQ4JLHBejzGIK/B762HKoQaFt59MNxv6+U7p+LzHuaPKn6r8vclDWjlTsnApoYBd7E/HdXqwWJlPvop0LnWaEms8G9zLpU+JP4vG7wiAnOXIA3Aj3d58PwAAnF2KxGP1At5zBGzfoNkUDlJijPALTV0KP3bL96HhC/zPC8MXoyOUcLqb6FhrfK+FZ5eznabl6NU98le2yb+yAheycVdk1q7E642zMfrNH9lT44615O96XmdUrzN98Zq/K2/sLNlm4uQ==</latexit> <latexit sha1_base64="LQjZYcsM1FBT7BVfqYAlPF9Sn5c=">AAACXXicfZBNSwMxEIbT9bt+tXrw4GWxCCJSdkXUkwh68CIqWBXcUmbTaQ1Nsksyq5alP8Wr/iZP/hXTD0GrOBB4eOcNM/PGqRSWguC94E1MTk3PzM4V5xcWl5ZL5ZUbm2SGY40nMjF3MViUQmONBEm8Sw2CiiXexp2Tfv/2EY0Vib6mbop1BW0tWoIDOalRKkdNlARHUQxmiI1SJagGg/J/QziCChvVZaNc2IuaCc8UauISrL0Pg5TqORgSXGKvGGUWU+AdaOO9Qw0KbT0f7N7zN53S9FuJcU+TP1C//8hBWdtVsXMqoAc73uuLf/ZiNTaZWof1XOg0I9R8OLiVSZ8Svx+M3xQGOcmuA+BGuN19/gAGOLn4isXoFN1xBs/doIsUDVBitvMITFsJ3XPHtqOdPv1nhOcvoyOXczie6m+42a2G+9X9q73K8cko8Vm2zjbYFgvZATtmZ+yS1RhnT+yFvbK3woc35S14S0OrVxj9WWU/ylv7BCXjt2k=</latexit>

    >¯ V (t)  <latexit sha1_base64="p0B7cx5cj5qkrIjmeYG55Kv3oy8=">AAACW3icfZDPattAEMZXcv+4alrbCT31stQU0lKMFIybYyA55BLqQm0HImFG67GzZHcldkelRvhJck0fqoe+S9eOC61TOrDw45tvmZkvL5V0FMc/grDx6PGTp81n0fO9Fy9b7c7+2BWVFTgShSrsZQ4OlTQ4IkkKL0uLoHOFk/zmdN2ffEXrZGG+0LLETMPCyLkUQF6atlvjQ3rHU4U8naEimLa7cS/eFH8IyRa6bFvDaSfop7NCVBoNCQXOXSVxSVkNlqRQuIrSymEJ4gYWeOXRgEaX1ZvNV/ytV2Z8Xlj/DPGN+uePGrRzS517pwa6dru9tfjPXq53JtP8OKulKStCI+4HzyvFqeDrWPhMWhSklh5AWOl35+IaLAjy4UVReob+OIsXftCnEi1QYd/XKdiFlmblj12kH9b0PyN8+2305HNOdlN9COOjXjLoDT73uyen28Sb7DV7ww5Zwj6yE3bOhmzEBKvYLbtj34OfYSOMwr17axhs/xywvyp89QuVkLWo</latexit> t = O(exp(exp( )) V (t) ⇡ p fa st <latexit sha1_base64="Z33XfrCdnuv0JpaVBFn5pCzzQh4=">AAACZHicfZDfSltBEMY3x9ZqqjYqhUJBlgbBFgnnFLFeCvaiN6UWTBQ8IczZTI6L+6+7c4rhEJ+mt+379AV8jm5iCm0sHVj48c03zM5XOCUDpenPRrL06PHyk5XV5tO19Y1nrc2tXrCVF9gVVll/UUBAJQ12SZLCC+cRdKHwvLg+mfbPv6IP0pozGjvsayiNHEkBFKVB60Vvj17zHJzz9obn4YunvAStYdBqp510VvwhZHNos3mdDjYbB/nQikqjIaEghMssddSvwZMUCifNvAroQFxDiZcRDWgM/Xp2woTvRmXIR9bHZ4jP1D8natAhjHURnRroKiz2puI/e4Ve2Eyjo34tjasIjbhfPKoUJ8un+fCh9ChIjSOA8DL+nYsr8CAopths5u8xHufxY1z0yaEHsv5NnYMvtTSTeGyZ70/pf0a4+W2MFHPOFlN9CL23neywc/j5oH18Mk98hb1kr9gey9g7dsw+sFPWZYLdsm/sO/vRuEvWku3k+b01acxnttlflez8ApWRueo=</latexit> [Bao [Soudry+ ] <latexit sha1_base64="JfMY6Mvt1u3czNJA7AWwaXD/x1E=">AAACcXicfVDRShtBFJ2sVdNobWKfii+DQdAqYbeI+ijoQ19EC0YFJ4S7k7txcGZ2mZlVw7pf0K/xVb+k39Ef6GyMoLH0wMCZc8/l3nviTArrwvB3LZj5MDs3X//YWFj8tPS52Vo+s2luOHZ5KlNzEYNFKTR2nXASLzKDoGKJ5/H1QVU/v0FjRapP3SjDnoKhFong4LzUb66xxAAvmAJ3FSf0dj3cKAt2//rP7st+sx12wjHoexJNSJtMcNJv1bbZIOW5Qu24BGsvozBzvQKME1xi2WC5xQz4NQzx0lMNCm2vGN9T0jWvDGiSGv+0o2P1dUcBytqRir2z2tNO1yrxn7VYTU12yV6vEDrLHWr+PDjJJXUprcKiA2GQOznyBLgRfnfKr8AH5nykjQY7RH+cwSM/6DhDAy413woGZqiELv2xQ7ZVsf8Z4e7F6JnPOZpO9T05+96Jdjo7P7fb+weTxOtkhaySdRKRXbJPfpAT0iWc/CIP5JE81f4EXwMarD5bg9qk5wt5g2DzL8NUvvo=</latexit> w(0) kw(0)k 方 非 比 方 8 1 6 2 Bao, H (2026). Non-asymptotic implicit bias of logistic regression at early-stage gradient descent dynamics. 7 9 📄 ¯>0 ⌧ w(t) w⇤ , kw(t)k kw⇤ k <latexit sha1_base64="VOobOnK2yFg3/q+MVGdJtHfHXog=">AAACVnicfZDdSgMxEIWz63/9q3rpzWIRRKTsiqhXUqgX3ogKVgW3yGw6raFJdklmxbL0NbzVx9KXEdNaQas4EPg4c8LMnCSTwlIYvnn+xOTU9MzsXGl+YXFpubyyemXT3HBs8FSm5iYBi1JobJAgiTeZQVCJxOukWx/0rx/QWJHqS+pl2FTQ0aItOJCT4jgBE7dQEhyFd+VKWA2HFfyGaAQVNqrzuxVvL26lPFeoiUuw9jYKM2oWYEhwif1SnFvMgHehg7cONSi0zWK4dD/YdEoraKfGPU3BUP3+owBlbU8lzqmA7u14byD+2UvU2GRqHzYLobOcUPPPwe1cBpQGg0SCljDISfYcADfC7R7wezDAyeVWKsXH6I4zeOoGnWVogFKzXcRgOkrovju2E+8M6D8jPH4ZHbmco/FUf8PVbjXar+5f7FVq9VHis2ydbbAtFrEDVmMn7Jw1GGcZe2LP7MV79d79KX/m0+p7oz9r7Ef55Q+8W7XP</latexit> ]
  71. r(t) := kw(t)k <latexit sha1_base64="lU9ZeMlw0wa94q+gsookqXGcyl0=">AAACcXicfVBdaxNBFJ2srdZYNbVP0pehodBWSXelVCkIgfTBF2kF0wS6Idyd3E2HzswuM3fVsOQX+Gt81V/i7+gf6Gw+QFPxwjCHc8/l3HuSXElHYfi7FjxYW3/4aONx/cnm02fPG1svLl1WWIFdkanM9hNwqKTBLklS2M8tgk4U9pKbTtXvfUHrZGY+0yTHgYaxkakUQJ4aNvZiDXSdpGUx3acDfvqeL4mvM+KIW/8NG82wFc6K3wfRAjTZoi6GW7XjeJSJQqMhocC5qyjMaVCCJSkUTutx4TAHcQNjvPLQgEY3KGf3TPmeZ0Y8zax/hviM/XOiBO3cRCdeWS3rVnsV+c9eolecKX03KKXJC0Ij5sZpoThlvAqLj6RFQWriAQgr/e5cXIMFQT7Sej0+Q3+cxY/e6DxHC5TZwzIGO9bSTP2x4/h1hf4nhG9LoUc+52g11fvg8k0rOmmdfDputjuLxDfYDttl+yxib1mbfWAXrMsE+85+sJ/sV+02eBnwYHcuDWqLmW32VwWv7gDIbL31</latexit> u(t) := w(t)/r(t) む Tangential

    + radial decomposition: w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む Derive ODE of tangential error Z <latexit sha1_base64="mdUv82MUYH6guKT+kuFQQJifY4g=">AAACS3icfVDLSgNBEJyNrxjfevQyGAQRCbtBYnIL6MGLqGBUdIP0TjpxcGZ2mZkVw5Iv8Kqf5Qf4Hd7Eg7Mxgora0FBUVdPdFSWCG+v7z15hbHxicqo4XZqZnZtfWFxaPjVxqhm2WCxifR6BQcEVtiy3As8TjSAjgWfRzW6un92iNjxWJ7afYFtCT/EuZ2AddXxxtVj2K76rWo3mIKj7gQONRr1abdBgKPl+mYzq6GrJ2w47MUslKssEGHMZ+IltZ6AtZwIHpTA1mAC7gR5eOqhAomlnw0sHdN0xHdqNtWtl6ZD9OpGBNKYvI+eUYK/NTy0nf9Ui+WOz7dbbGVdJalGxj8XdVFAb0zwG2uEamRV9B4Bp7m6n7Bo0MOvCKpXCPXTPaTxwiw4T1GBjvZmFoHuSq4F7thdu5eg/I9x9Gh1yOX+GSf8Gp9VKUKvUjrfLzd1R4kWyStbIBgnIDmmSfXJEWoQRJPfkgTx6T96L9+q9fVgL3mhmhXyrwsQ7CW+zkQ==</latexit> ⇠ 1/t log t <latexit sha1_base64="+KQO3bCW4mewCdd+OjKkYLp5Wec=">AAACWHicfZDbShxBEIZrJwd1Y+Ihl7lpsgSChM2MiHop6EVuRIWsCs5Ganprx8Y+DN014jLse+TWvFV8GnvXDZg1pKDh46+/qaq/qLQKnKa/W8mLl69eLywutd8sv323srq2fhpc7SX1pNPOnxcYSCtLPVas6bzyhKbQdFZc70/6Zzfkg3L2O48q6hssrRoqiRylH3lQRmRfOdeuFHy52km76bTEc8hm0IFZHV+utbbygZO1IctSYwgXWVpxv0HPSmoat/M6UIXyGku6iGjRUOg307XH4lNUBmLofHyWxVR9+qNBE8LIFNFpkK/CfG8i/rNXmLnJPNztN8pWNZOVj4OHtRbsxCQTMVCeJOtRBJRexd2FvEKPkmNy7XZ+QPE4T4dx0FFFHtn5jSZHXxplx/HYMv8yof8Z8faPMVLMOZtP9Tmcbnaz7e72yVZnb3+W+CJ8gI/wGTLYgT34BsfQAwkefsId/GrdJ5AsJEuP1qQ1+/Me/qpk/QGp6bVK</latexit> ? む Control hZ↵, Pu u⇤ i at early stage <latexit sha1_base64="i6WJC5NFnLke0SgTAnGIXphVIEw=">AAACjnicfVBdaxQxFM2OX3X92uqjL8FFkFKWmVJWX0pL60NfxBXcttisw53sndnQTBKSjLgM83/6a3y1/TdmtiPoVrwQcnLuudyckxkpnI/j61505+69+w82HvYfPX7y9Nlg8/mJ05XlOOVaanuWgUMpFE698BLPjEUoM4mn2cVR2z/9htYJrT77pcFZCYUSueDgA5UODpkEVUikX1im5dwty3AxkGYB23SS1qwEv8hyWjVfmUFrunddNekWs6vRdDCMR/Gq6G2QdGBIupqkm71dNte8KlF5LsG58yQ2flaD9YJLbPqscmiAX0CB5wEqKNHN6pXZhr4OzJzm2oajPF2xf07UULrWRlC2f3XrvZb8Zy8r1zb7/N2sFspUHhW/WZxXknpN2yTpXFjkXi4DAG5F+DvlC7DAfci732fvMZiz+CEs+hiiA6/tVs3AFqVQTTBbsO0W/U8I338LAwo5J+up3gYnO6NkPBp/2h0eHHWJb5CX5BV5QxLylhyQYzIhU8LJJflBfpKraBCNo71o/0Ya9bqZF+Svio5/AWgEy1c=</latexit> 8 9 Proof sketch <latexit sha1_base64="aaWHJn5I3Smwh1s+JpcIMsIm12Q=">AAACZXicfZDRShtBFIYnW6s2Vo21FEovHBoKtpSwW0RFEAR70RupglHBDeHs5GwcnJldZs5Wwxp8Gm/1eXyCvkZnYwo1Fg8MfPznH845f5Ir6SgM72vBi6mX0zOzr+pzr+cXFhtLb45cVliBbZGpzJ4k4FBJg22SpPAktwg6UXicnO9W/eNfaJ3MzCENcuxo6BuZSgHkpW7jvV2lz3xrm8dXsQY6S9LyYuil+KrbaIatcFT8KURjaLJx7XeXamtxLxOFRkNCgXOnUZhTpwRLUigc1uPCYQ7iHPp46tGARtcpRzcM+Sev9HiaWf8M8ZH6748StHMDnXhntaeb7FXif3uJnphM6WanlCYvCI14GJwWilPGq4B4T1oUpAYeQFjpd+fiDCwI8jHW6/F39MdZ3PODfuZogTL7pYzB9rU0Q39sP/5a0XNGuPxr9ORzjiZTfQpH31rRemv9YK25sztOfJZ9YB/ZKovYBtthP9g+azPBrtkNu2V3td/BfPA2ePdgDWrjP8vsUQUrfwBV5Lm+</latexit> Z↵ <latexit sha1_base64="ntsWT+hokTUIU4cKTIGYy8Kt/zE=">AAACXnicfVBdSxtBFJ1s1WqqNdoXwZfBUChSwm6QNHkT7IMvRQVjQt0Q7k5ukiHzsczMFsOSv+Kr/Ut986d0NiagQb0wzOGcc7n3niQV3Low/FcKPqytb3zc3Cp/2t75vFvZ27+xOjMM20wLbboJWBRcYdtxJ7CbGgSZCOwkk7NC7/xBY7lW126aYk/CSPEhZ+A81a/s/44TLQZ2Kv0Xg0jH0K9Uw1roq9GgBYiaYeRBq9Ws11s0mkthWCWLuuzvlU7igWaZROWYAGtvozB1vRyM40zgrBxnFlNgExjhrYcKJNpePl9+Rr96ZkCH2vinHJ2zzztykLZYzzsluLFd1QryVS2RK5PdsNnLuUozh4o9DR5mgjpNi2TogBtkTkw9AGa4352yMRhgzudXLsc/0R9n8JcfdJGiAafNcR6DGUmuZv7YUfy9QO8Z4W5p9MjnvAyTvg1u6rWoUWtcnVRPzxaJb5JDckS+kYj8IKfknFySNmHkjtyTB/K39BhsBDvB7pM1KC16vpAXFRz8BxpnuGQ=</latexit>
  72. r(t) := kw(t)k <latexit sha1_base64="lU9ZeMlw0wa94q+gsookqXGcyl0=">AAACcXicfVBdaxNBFJ2srdZYNbVP0pehodBWSXelVCkIgfTBF2kF0wS6Idyd3E2HzswuM3fVsOQX+Gt81V/i7+gf6Gw+QFPxwjCHc8/l3HuSXElHYfi7FjxYW3/4aONx/cnm02fPG1svLl1WWIFdkanM9hNwqKTBLklS2M8tgk4U9pKbTtXvfUHrZGY+0yTHgYaxkakUQJ4aNvZiDXSdpGUx3acDfvqeL4mvM+KIW/8NG82wFc6K3wfRAjTZoi6GW7XjeJSJQqMhocC5qyjMaVCCJSkUTutx4TAHcQNjvPLQgEY3KGf3TPmeZ0Y8zax/hviM/XOiBO3cRCdeWS3rVnsV+c9eolecKX03KKXJC0Ij5sZpoThlvAqLj6RFQWriAQgr/e5cXIMFQT7Sej0+Q3+cxY/e6DxHC5TZwzIGO9bSTP2x4/h1hf4nhG9LoUc+52g11fvg8k0rOmmdfDputjuLxDfYDttl+yxib1mbfWAXrMsE+85+sJ/sV+02eBnwYHcuDWqLmW32VwWv7gDIbL31</latexit> u(t) := w(t)/r(t) む Tangential

    + radial decomposition: w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む Derive ODE of tangential error ⇠ 1/t log t <latexit sha1_base64="+KQO3bCW4mewCdd+OjKkYLp5Wec=">AAACWHicfZDbShxBEIZrJwd1Y+Ihl7lpsgSChM2MiHop6EVuRIWsCs5Ganprx8Y+DN014jLse+TWvFV8GnvXDZg1pKDh46+/qaq/qLQKnKa/W8mLl69eLywutd8sv323srq2fhpc7SX1pNPOnxcYSCtLPVas6bzyhKbQdFZc70/6Zzfkg3L2O48q6hssrRoqiRylH3lQRmRfOdeuFHy52km76bTEc8hm0IFZHV+utbbygZO1IctSYwgXWVpxv0HPSmoat/M6UIXyGku6iGjRUOg307XH4lNUBmLofHyWxVR9+qNBE8LIFNFpkK/CfG8i/rNXmLnJPNztN8pWNZOVj4OHtRbsxCQTMVCeJOtRBJRexd2FvEKPkmNy7XZ+QPE4T4dx0FFFHtn5jSZHXxplx/HYMv8yof8Z8faPMVLMOZtP9Tmcbnaz7e72yVZnb3+W+CJ8gI/wGTLYgT34BsfQAwkefsId/GrdJ5AsJEuP1qQ1+/Me/qpk/QGp6bVK</latexit> ? む Control hZ↵, Pu u⇤ i at early stage <latexit sha1_base64="i6WJC5NFnLke0SgTAnGIXphVIEw=">AAACjnicfVBdaxQxFM2OX3X92uqjL8FFkFKWmVJWX0pL60NfxBXcttisw53sndnQTBKSjLgM83/6a3y1/TdmtiPoVrwQcnLuudyckxkpnI/j61505+69+w82HvYfPX7y9Nlg8/mJ05XlOOVaanuWgUMpFE698BLPjEUoM4mn2cVR2z/9htYJrT77pcFZCYUSueDgA5UODpkEVUikX1im5dwty3AxkGYB23SS1qwEv8hyWjVfmUFrunddNekWs6vRdDCMR/Gq6G2QdGBIupqkm71dNte8KlF5LsG58yQ2flaD9YJLbPqscmiAX0CB5wEqKNHN6pXZhr4OzJzm2oajPF2xf07UULrWRlC2f3XrvZb8Zy8r1zb7/N2sFspUHhW/WZxXknpN2yTpXFjkXi4DAG5F+DvlC7DAfci732fvMZiz+CEs+hiiA6/tVs3AFqVQTTBbsO0W/U8I338LAwo5J+up3gYnO6NkPBp/2h0eHHWJb5CX5BV5QxLylhyQYzIhU8LJJflBfpKraBCNo71o/0Ya9bqZF+Svio5/AWgEy1c=</latexit> 9 9 Proof sketch <latexit sha1_base64="aaWHJn5I3Smwh1s+JpcIMsIm12Q=">AAACZXicfZDRShtBFIYnW6s2Vo21FEovHBoKtpSwW0RFEAR70RupglHBDeHs5GwcnJldZs5Wwxp8Gm/1eXyCvkZnYwo1Fg8MfPznH845f5Ir6SgM72vBi6mX0zOzr+pzr+cXFhtLb45cVliBbZGpzJ4k4FBJg22SpPAktwg6UXicnO9W/eNfaJ3MzCENcuxo6BuZSgHkpW7jvV2lz3xrm8dXsQY6S9LyYuil+KrbaIatcFT8KURjaLJx7XeXamtxLxOFRkNCgXOnUZhTpwRLUigc1uPCYQ7iHPp46tGARtcpRzcM+Sev9HiaWf8M8ZH6748StHMDnXhntaeb7FXif3uJnphM6WanlCYvCI14GJwWilPGq4B4T1oUpAYeQFjpd+fiDCwI8jHW6/F39MdZ3PODfuZogTL7pYzB9rU0Q39sP/5a0XNGuPxr9ORzjiZTfQpH31rRemv9YK25sztOfJZ9YB/ZKovYBtthP9g+azPBrtkNu2V3td/BfPA2ePdgDWrjP8vsUQUrfwBV5Lm+</latexit>
  73. r(t) := kw(t)k <latexit sha1_base64="lU9ZeMlw0wa94q+gsookqXGcyl0=">AAACcXicfVBdaxNBFJ2srdZYNbVP0pehodBWSXelVCkIgfTBF2kF0wS6Idyd3E2HzswuM3fVsOQX+Gt81V/i7+gf6Gw+QFPxwjCHc8/l3HuSXElHYfi7FjxYW3/4aONx/cnm02fPG1svLl1WWIFdkanM9hNwqKTBLklS2M8tgk4U9pKbTtXvfUHrZGY+0yTHgYaxkakUQJ4aNvZiDXSdpGUx3acDfvqeL4mvM+KIW/8NG82wFc6K3wfRAjTZoi6GW7XjeJSJQqMhocC5qyjMaVCCJSkUTutx4TAHcQNjvPLQgEY3KGf3TPmeZ0Y8zax/hviM/XOiBO3cRCdeWS3rVnsV+c9eolecKX03KKXJC0Ij5sZpoThlvAqLj6RFQWriAQgr/e5cXIMFQT7Sej0+Q3+cxY/e6DxHC5TZwzIGO9bSTP2x4/h1hf4nhG9LoUc+52g11fvg8k0rOmmdfDputjuLxDfYDttl+yxib1mbfWAXrMsE+85+sJ/sV+02eBnwYHcuDWqLmW32VwWv7gDIbL31</latexit> u(t) := w(t)/r(t) む Tangential

    + radial decomposition: w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む Derive ODE of tangential error Z <latexit sha1_base64="mdUv82MUYH6guKT+kuFQQJifY4g=">AAACS3icfVDLSgNBEJyNrxjfevQyGAQRCbtBYnIL6MGLqGBUdIP0TjpxcGZ2mZkVw5Iv8Kqf5Qf4Hd7Eg7Mxgora0FBUVdPdFSWCG+v7z15hbHxicqo4XZqZnZtfWFxaPjVxqhm2WCxifR6BQcEVtiy3As8TjSAjgWfRzW6un92iNjxWJ7afYFtCT/EuZ2AddXxxtVj2K76rWo3mIKj7gQONRr1abdBgKPl+mYzq6GrJ2w47MUslKssEGHMZ+IltZ6AtZwIHpTA1mAC7gR5eOqhAomlnw0sHdN0xHdqNtWtl6ZD9OpGBNKYvI+eUYK/NTy0nf9Ui+WOz7dbbGVdJalGxj8XdVFAb0zwG2uEamRV9B4Bp7m6n7Bo0MOvCKpXCPXTPaTxwiw4T1GBjvZmFoHuSq4F7thdu5eg/I9x9Gh1yOX+GSf8Gp9VKUKvUjrfLzd1R4kWyStbIBgnIDmmSfXJEWoQRJPfkgTx6T96L9+q9fVgL3mhmhXyrwsQ7CW+zkQ==</latexit> ⇠ 1/t log t <latexit sha1_base64="+KQO3bCW4mewCdd+OjKkYLp5Wec=">AAACWHicfZDbShxBEIZrJwd1Y+Ihl7lpsgSChM2MiHop6EVuRIWsCs5Ganprx8Y+DN014jLse+TWvFV8GnvXDZg1pKDh46+/qaq/qLQKnKa/W8mLl69eLywutd8sv323srq2fhpc7SX1pNPOnxcYSCtLPVas6bzyhKbQdFZc70/6Zzfkg3L2O48q6hssrRoqiRylH3lQRmRfOdeuFHy52km76bTEc8hm0IFZHV+utbbygZO1IctSYwgXWVpxv0HPSmoat/M6UIXyGku6iGjRUOg307XH4lNUBmLofHyWxVR9+qNBE8LIFNFpkK/CfG8i/rNXmLnJPNztN8pWNZOVj4OHtRbsxCQTMVCeJOtRBJRexd2FvEKPkmNy7XZ+QPE4T4dx0FFFHtn5jSZHXxplx/HYMv8yof8Z8faPMVLMOZtP9Tmcbnaz7e72yVZnb3+W+CJ8gI/wGTLYgT34BsfQAwkefsId/GrdJ5AsJEuP1qQ1+/Me/qpk/QGp6bVK</latexit> ? む Control hZ↵, Pu u⇤ i at early stage 0 <latexit sha1_base64="i6WJC5NFnLke0SgTAnGIXphVIEw=">AAACjnicfVBdaxQxFM2OX3X92uqjL8FFkFKWmVJWX0pL60NfxBXcttisw53sndnQTBKSjLgM83/6a3y1/TdmtiPoVrwQcnLuudyckxkpnI/j61505+69+w82HvYfPX7y9Nlg8/mJ05XlOOVaanuWgUMpFE698BLPjEUoM4mn2cVR2z/9htYJrT77pcFZCYUSueDgA5UODpkEVUikX1im5dwty3AxkGYB23SS1qwEv8hyWjVfmUFrunddNekWs6vRdDCMR/Gq6G2QdGBIupqkm71dNte8KlF5LsG58yQ2flaD9YJLbPqscmiAX0CB5wEqKNHN6pXZhr4OzJzm2oajPF2xf07UULrWRlC2f3XrvZb8Zy8r1zb7/N2sFspUHhW/WZxXknpN2yTpXFjkXi4DAG5F+DvlC7DAfci732fvMZiz+CEs+hiiA6/tVs3AFqVQTTBbsO0W/U8I338LAwo5J+up3gYnO6NkPBp/2h0eHHWJb5CX5BV5QxLylhyQYzIhU8LJJflBfpKraBCNo71o/0Ya9bqZF+Svio5/AWgEy1c=</latexit> 0 1 Proof sketch <latexit sha1_base64="aaWHJn5I3Smwh1s+JpcIMsIm12Q=">AAACZXicfZDRShtBFIYnW6s2Vo21FEovHBoKtpSwW0RFEAR70RupglHBDeHs5GwcnJldZs5Wwxp8Gm/1eXyCvkZnYwo1Fg8MfPznH845f5Ir6SgM72vBi6mX0zOzr+pzr+cXFhtLb45cVliBbZGpzJ4k4FBJg22SpPAktwg6UXicnO9W/eNfaJ3MzCENcuxo6BuZSgHkpW7jvV2lz3xrm8dXsQY6S9LyYuil+KrbaIatcFT8KURjaLJx7XeXamtxLxOFRkNCgXOnUZhTpwRLUigc1uPCYQ7iHPp46tGARtcpRzcM+Sev9HiaWf8M8ZH6748StHMDnXhntaeb7FXif3uJnphM6WanlCYvCI14GJwWilPGq4B4T1oUpAYeQFjpd+fiDCwI8jHW6/F39MdZ3PODfuZogTL7pYzB9rU0Q39sP/5a0XNGuPxr9ORzjiZTfQpH31rRemv9YK25sztOfJZ9YB/ZKovYBtthP9g+azPBrtkNu2V3td/BfPA2ePdgDWrjP8vsUQUrfwBV5Lm+</latexit> Z↵ <latexit sha1_base64="ntsWT+hokTUIU4cKTIGYy8Kt/zE=">AAACXnicfVBdSxtBFJ1s1WqqNdoXwZfBUChSwm6QNHkT7IMvRQVjQt0Q7k5ukiHzsczMFsOSv+Kr/Ut986d0NiagQb0wzOGcc7n3niQV3Low/FcKPqytb3zc3Cp/2t75vFvZ27+xOjMM20wLbboJWBRcYdtxJ7CbGgSZCOwkk7NC7/xBY7lW126aYk/CSPEhZ+A81a/s/44TLQZ2Kv0Xg0jH0K9Uw1roq9GgBYiaYeRBq9Ws11s0mkthWCWLuuzvlU7igWaZROWYAGtvozB1vRyM40zgrBxnFlNgExjhrYcKJNpePl9+Rr96ZkCH2vinHJ2zzztykLZYzzsluLFd1QryVS2RK5PdsNnLuUozh4o9DR5mgjpNi2TogBtkTkw9AGa4352yMRhgzudXLsc/0R9n8JcfdJGiAafNcR6DGUmuZv7YUfy9QO8Z4W5p9MjnvAyTvg1u6rWoUWtcnVRPzxaJb5JDckS+kYj8IKfknFySNmHkjtyTB/K39BhsBDvB7pM1KC16vpAXFRz8BxpnuGQ=</latexit>
  74. r(t) := kw(t)k <latexit sha1_base64="lU9ZeMlw0wa94q+gsookqXGcyl0=">AAACcXicfVBdaxNBFJ2srdZYNbVP0pehodBWSXelVCkIgfTBF2kF0wS6Idyd3E2HzswuM3fVsOQX+Gt81V/i7+gf6Gw+QFPxwjCHc8/l3HuSXElHYfi7FjxYW3/4aONx/cnm02fPG1svLl1WWIFdkanM9hNwqKTBLklS2M8tgk4U9pKbTtXvfUHrZGY+0yTHgYaxkakUQJ4aNvZiDXSdpGUx3acDfvqeL4mvM+KIW/8NG82wFc6K3wfRAjTZoi6GW7XjeJSJQqMhocC5qyjMaVCCJSkUTutx4TAHcQNjvPLQgEY3KGf3TPmeZ0Y8zax/hviM/XOiBO3cRCdeWS3rVnsV+c9eolecKX03KKXJC0Ij5sZpoThlvAqLj6RFQWriAQgr/e5cXIMFQT7Sej0+Q3+cxY/e6DxHC5TZwzIGO9bSTP2x4/h1hf4nhG9LoUc+52g11fvg8k0rOmmdfDputjuLxDfYDttl+yxib1mbfWAXrMsE+85+sJ/sV+02eBnwYHcuDWqLmW32VwWv7gDIbL31</latexit> u(t) := w(t)/r(t) む Tangential

    + radial decomposition: w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む Derive ODE of tangential error ⇠ 1/t log t <latexit sha1_base64="+KQO3bCW4mewCdd+OjKkYLp5Wec=">AAACWHicfZDbShxBEIZrJwd1Y+Ihl7lpsgSChM2MiHop6EVuRIWsCs5Ganprx8Y+DN014jLse+TWvFV8GnvXDZg1pKDh46+/qaq/qLQKnKa/W8mLl69eLywutd8sv323srq2fhpc7SX1pNPOnxcYSCtLPVas6bzyhKbQdFZc70/6Zzfkg3L2O48q6hssrRoqiRylH3lQRmRfOdeuFHy52km76bTEc8hm0IFZHV+utbbygZO1IctSYwgXWVpxv0HPSmoat/M6UIXyGku6iGjRUOg307XH4lNUBmLofHyWxVR9+qNBE8LIFNFpkK/CfG8i/rNXmLnJPNztN8pWNZOVj4OHtRbsxCQTMVCeJOtRBJRexd2FvEKPkmNy7XZ+QPE4T4dx0FFFHtn5jSZHXxplx/HYMv8yof8Z8faPMVLMOZtP9Tmcbnaz7e72yVZnb3+W+CJ8gI/wGTLYgT34BsfQAwkefsId/GrdJ5AsJEuP1qQ1+/Me/qpk/QGp6bVK</latexit> ? む Control hZ↵, Pu u⇤ i at early stage <latexit sha1_base64="i6WJC5NFnLke0SgTAnGIXphVIEw=">AAACjnicfVBdaxQxFM2OX3X92uqjL8FFkFKWmVJWX0pL60NfxBXcttisw53sndnQTBKSjLgM83/6a3y1/TdmtiPoVrwQcnLuudyckxkpnI/j61505+69+w82HvYfPX7y9Nlg8/mJ05XlOOVaanuWgUMpFE698BLPjEUoM4mn2cVR2z/9htYJrT77pcFZCYUSueDgA5UODpkEVUikX1im5dwty3AxkGYB23SS1qwEv8hyWjVfmUFrunddNekWs6vRdDCMR/Gq6G2QdGBIupqkm71dNte8KlF5LsG58yQ2flaD9YJLbPqscmiAX0CB5wEqKNHN6pXZhr4OzJzm2oajPF2xf07UULrWRlC2f3XrvZb8Zy8r1zb7/N2sFspUHhW/WZxXknpN2yTpXFjkXi4DAG5F+DvlC7DAfci732fvMZiz+CEs+hiiA6/tVs3AFqVQTTBbsO0W/U8I338LAwo5J+up3gYnO6NkPBp/2h0eHHWJb5CX5BV5QxLylhyQYzIhU8LJJflBfpKraBCNo71o/0Ya9bqZF+Svio5/AWgEy1c=</latexit> Z↵ <latexit sha1_base64="ntsWT+hokTUIU4cKTIGYy8Kt/zE=">AAACXnicfVBdSxtBFJ1s1WqqNdoXwZfBUChSwm6QNHkT7IMvRQVjQt0Q7k5ukiHzsczMFsOSv+Kr/Ut986d0NiagQb0wzOGcc7n3niQV3Low/FcKPqytb3zc3Cp/2t75vFvZ27+xOjMM20wLbboJWBRcYdtxJ7CbGgSZCOwkk7NC7/xBY7lW126aYk/CSPEhZ+A81a/s/44TLQZ2Kv0Xg0jH0K9Uw1roq9GgBYiaYeRBq9Ws11s0mkthWCWLuuzvlU7igWaZROWYAGtvozB1vRyM40zgrBxnFlNgExjhrYcKJNpePl9+Rr96ZkCH2vinHJ2zzztykLZYzzsluLFd1QryVS2RK5PdsNnLuUozh4o9DR5mgjpNi2TogBtkTkw9AGa4352yMRhgzudXLsc/0R9n8JcfdJGiAafNcR6DGUmuZv7YUfy9QO8Z4W5p9MjnvAyTvg1u6rWoUWtcnVRPzxaJb5JDckS+kYj8IKfknFySNmHkjtyTB/K39BhsBDvB7pM1KC16vpAXFRz8BxpnuGQ=</latexit> u⇤ <latexit sha1_base64="IL1H7dLQPme3LRQ357YEOrGwqSE=">AAACVnicfVBNSyNBEO2ZNZqNX3E97qUxCCISZoKMyU3IHvYi64JRIRNCTacmNnb3DN09YhjyN7zu/qzdPyP2xAi7ohYUPN57RVW9JBfc2CD46/mfVmqra/XPjfWNza3t5s6XS5MVmuGAZSLT1wkYFFzhwHIr8DrXCDIReJXc9iv96g614Zm6sLMcRxKmiqecgXVUHEuwN0laFvPx4bjZCtqBqyiiFQi7QehAr9ftdHo0XEhB0CLLOh/veMfxJGOFRGWZAGOGYZDbUQnaciZw3ogLgzmwW5ji0EEFEs2oXBw9p/uOmdA0066VpQv234kSpDEzmThndaR5rVXkm1oiX222aXdUcpUXFhV7XpwWgtqMVonQCdfIrJg5AExzdztlN6CBWZdboxF/Q/ecxjO36EeOGmymD8sY9FRyNXfPTuOjCn1khPsXo0Mu55cw6fvgstMOo3b087h12l8mXidfyR45ICE5IafkOzknA8JITh7IL/Lb++M9+jV/7dnqe8uZXfJf+c0n4WK2aw==</latexit> u 1 <latexit sha1_base64="61qfHGqKCNmsdzYd3BnJfVj0xwo=">AAACVHicfVBNS8NAEN3E7/qtRy+LRRCRkhSp7U3QgxdRwapoiky2k7q4uwm7G7GE/guv+rME/4sHN7WCijow8HjvDTPz4kxwY4Pg1fPHxicmp6ZnKrNz8wuLS8sr5ybNNcM2S0WqL2MwKLjCtuVW4GWmEWQs8CK+2y/1i3vUhqfqzPYz7EjoKZ5wBtZRV5EEexsnRT64WaoGtcBVo0FLEDaD0IFWq1mvt2g4lIKgSkZ1crPs7UTdlOUSlWUCjLkOg8x2CtCWM4GDSpQbzIDdQQ+vHVQg0XSK4ckDuuGYLk1S7VpZOmS/ThQgjenL2DnLE81PrSR/1WL5Y7NNmp2Cqyy3qNjH4iQX1Ka0zIN2uUZmRd8BYJq72ym7BQ3MutQqlegA3XMaj9yi4ww12FRvFRHonuRq4J7tRdsl+s8ID59Gh1zOn2HSv8F5vRY2ao3Tnere/ijxabJG1skmCcku2SOH5IS0CSOKPJIn8uy9eG/+mD/xYfW90cwq+Vb+wjuJK7XO</latexit> 0 1 Proof sketch <latexit sha1_base64="aaWHJn5I3Smwh1s+JpcIMsIm12Q=">AAACZXicfZDRShtBFIYnW6s2Vo21FEovHBoKtpSwW0RFEAR70RupglHBDeHs5GwcnJldZs5Wwxp8Gm/1eXyCvkZnYwo1Fg8MfPznH845f5Ir6SgM72vBi6mX0zOzr+pzr+cXFhtLb45cVliBbZGpzJ4k4FBJg22SpPAktwg6UXicnO9W/eNfaJ3MzCENcuxo6BuZSgHkpW7jvV2lz3xrm8dXsQY6S9LyYuil+KrbaIatcFT8KURjaLJx7XeXamtxLxOFRkNCgXOnUZhTpwRLUigc1uPCYQ7iHPp46tGARtcpRzcM+Sev9HiaWf8M8ZH6748StHMDnXhntaeb7FXif3uJnphM6WanlCYvCI14GJwWilPGq4B4T1oUpAYeQFjpd+fiDCwI8jHW6/F39MdZ3PODfuZogTL7pYzB9rU0Q39sP/5a0XNGuPxr9ORzjiZTfQpH31rRemv9YK25sztOfJZ9YB/ZKovYBtthP9g+azPBrtkNu2V3td/BfPA2ePdgDWrjP8vsUQUrfwBV5Lm+</latexit>
  75. r(t) := kw(t)k <latexit sha1_base64="lU9ZeMlw0wa94q+gsookqXGcyl0=">AAACcXicfVBdaxNBFJ2srdZYNbVP0pehodBWSXelVCkIgfTBF2kF0wS6Idyd3E2HzswuM3fVsOQX+Gt81V/i7+gf6Gw+QFPxwjCHc8/l3HuSXElHYfi7FjxYW3/4aONx/cnm02fPG1svLl1WWIFdkanM9hNwqKTBLklS2M8tgk4U9pKbTtXvfUHrZGY+0yTHgYaxkakUQJ4aNvZiDXSdpGUx3acDfvqeL4mvM+KIW/8NG82wFc6K3wfRAjTZoi6GW7XjeJSJQqMhocC5qyjMaVCCJSkUTutx4TAHcQNjvPLQgEY3KGf3TPmeZ0Y8zax/hviM/XOiBO3cRCdeWS3rVnsV+c9eolecKX03KKXJC0Ij5sZpoThlvAqLj6RFQWriAQgr/e5cXIMFQT7Sej0+Q3+cxY/e6DxHC5TZwzIGO9bSTP2x4/h1hf4nhG9LoUc+52g11fvg8k0rOmmdfDputjuLxDfYDttl+yxib1mbfWAXrMsE+85+sJ/sV+02eBnwYHcuDWqLmW32VwWv7gDIbL31</latexit> u(t) := w(t)/r(t) む Tangential

    + radial decomposition: w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む Derive ODE of tangential error ⇠ 1/t log t <latexit sha1_base64="+KQO3bCW4mewCdd+OjKkYLp5Wec=">AAACWHicfZDbShxBEIZrJwd1Y+Ihl7lpsgSChM2MiHop6EVuRIWsCs5Ganprx8Y+DN014jLse+TWvFV8GnvXDZg1pKDh46+/qaq/qLQKnKa/W8mLl69eLywutd8sv323srq2fhpc7SX1pNPOnxcYSCtLPVas6bzyhKbQdFZc70/6Zzfkg3L2O48q6hssrRoqiRylH3lQRmRfOdeuFHy52km76bTEc8hm0IFZHV+utbbygZO1IctSYwgXWVpxv0HPSmoat/M6UIXyGku6iGjRUOg307XH4lNUBmLofHyWxVR9+qNBE8LIFNFpkK/CfG8i/rNXmLnJPNztN8pWNZOVj4OHtRbsxCQTMVCeJOtRBJRexd2FvEKPkmNy7XZ+QPE4T4dx0FFFHtn5jSZHXxplx/HYMv8yof8Z8faPMVLMOZtP9Tmcbnaz7e72yVZnb3+W+CJ8gI/wGTLYgT34BsfQAwkefsId/GrdJ5AsJEuP1qQ1+/Me/qpk/QGp6bVK</latexit> ? む Control hZ↵, Pu u⇤ i at early stage <latexit sha1_base64="i6WJC5NFnLke0SgTAnGIXphVIEw=">AAACjnicfVBdaxQxFM2OX3X92uqjL8FFkFKWmVJWX0pL60NfxBXcttisw53sndnQTBKSjLgM83/6a3y1/TdmtiPoVrwQcnLuudyckxkpnI/j61505+69+w82HvYfPX7y9Nlg8/mJ05XlOOVaanuWgUMpFE698BLPjEUoM4mn2cVR2z/9htYJrT77pcFZCYUSueDgA5UODpkEVUikX1im5dwty3AxkGYB23SS1qwEv8hyWjVfmUFrunddNekWs6vRdDCMR/Gq6G2QdGBIupqkm71dNte8KlF5LsG58yQ2flaD9YJLbPqscmiAX0CB5wEqKNHN6pXZhr4OzJzm2oajPF2xf07UULrWRlC2f3XrvZb8Zy8r1zb7/N2sFspUHhW/WZxXknpN2yTpXFjkXi4DAG5F+DvlC7DAfci732fvMZiz+CEs+hiiA6/tVs3AFqVQTTBbsO0W/U8I338LAwo5J+up3gYnO6NkPBp/2h0eHHWJb5CX5BV5QxLylhyQYzIhU8LJJflBfpKraBCNo71o/0Ya9bqZF+Svio5/AWgEy1c=</latexit> Z↵ <latexit sha1_base64="ntsWT+hokTUIU4cKTIGYy8Kt/zE=">AAACXnicfVBdSxtBFJ1s1WqqNdoXwZfBUChSwm6QNHkT7IMvRQVjQt0Q7k5ukiHzsczMFsOSv+Kr/Ut986d0NiagQb0wzOGcc7n3niQV3Low/FcKPqytb3zc3Cp/2t75vFvZ27+xOjMM20wLbboJWBRcYdtxJ7CbGgSZCOwkk7NC7/xBY7lW126aYk/CSPEhZ+A81a/s/44TLQZ2Kv0Xg0jH0K9Uw1roq9GgBYiaYeRBq9Ws11s0mkthWCWLuuzvlU7igWaZROWYAGtvozB1vRyM40zgrBxnFlNgExjhrYcKJNpePl9+Rr96ZkCH2vinHJ2zzztykLZYzzsluLFd1QryVS2RK5PdsNnLuUozh4o9DR5mgjpNi2TogBtkTkw9AGa4352yMRhgzudXLsc/0R9n8JcfdJGiAafNcR6DGUmuZv7YUfy9QO8Z4W5p9MjnvAyTvg1u6rWoUWtcnVRPzxaJb5JDckS+kYj8IKfknFySNmHkjtyTB/K39BhsBDvB7pM1KC16vpAXFRz8BxpnuGQ=</latexit> ? Pu u ⇤ <latexit sha1_base64="0Fu26QSblnPQfllr+h6sSpOsW90=">AAACa3icfVDRShtBFJ1srdpYNda3KjgYBBEJu0Fi8ibYh74UIxgV3HS5O7kbB2dml5nZ0rDsY7+mr/ox/Qj/wdkYwYp6YeDMOedy7z1xJrixvv+v5n2Y+zi/sPipvvR5eWW1sfbl3KS5ZjhgqUj1ZQwGBVc4sNwKvMw0gowFXsQ3x5V+8Qu14ak6s5MMhxLGiiecgXVU1NjqR0UowV7HCc3Ln2GGOpv9i7yM9qJG02/5rjodWoGg6wcO9HrddrtHg6nk+00yq360VjsIRynLJSrLBBhzFfiZHRagLWcCy3qYG8yA3cAYrxxUINEMi+klJd1xzIgmqXZPWTpln3cUII2ZyNg5qyXNS60iX9Vi+WKyTbrDgqsst6jY4+AkF9SmtIqJjrhGZsXEAWCau90puwYNzLow6/XwG7rjNP5wg05cZmBTvVeEoMeSq9IdOw73K/SeEX4/GR1yOT+FSd8G5+1W0Gl1Tg+aR8ezxBfJBtkmuyQgh+SIfCd9MiCM/CF/yS25q917695Xb/PR6tVmPevkv/J2HgDhYb3W</latexit> u 2 <latexit sha1_base64="61qfHGqKCNmsdzYd3BnJfVj0xwo=">AAACVHicfVBNS8NAEN3E7/qtRy+LRRCRkhSp7U3QgxdRwapoiky2k7q4uwm7G7GE/guv+rME/4sHN7WCijow8HjvDTPz4kxwY4Pg1fPHxicmp6ZnKrNz8wuLS8sr5ybNNcM2S0WqL2MwKLjCtuVW4GWmEWQs8CK+2y/1i3vUhqfqzPYz7EjoKZ5wBtZRV5EEexsnRT64WaoGtcBVo0FLEDaD0IFWq1mvt2g4lIKgSkZ1crPs7UTdlOUSlWUCjLkOg8x2CtCWM4GDSpQbzIDdQQ+vHVQg0XSK4ckDuuGYLk1S7VpZOmS/ThQgjenL2DnLE81PrSR/1WL5Y7NNmp2Cqyy3qNjH4iQX1Ka0zIN2uUZmRd8BYJq72ym7BQ3MutQqlegA3XMaj9yi4ww12FRvFRHonuRq4J7tRdsl+s8ID59Gh1zOn2HSv8F5vRY2ao3Tnere/ijxabJG1skmCcku2SOH5IS0CSOKPJIn8uy9eG/+mD/xYfW90cwq+Vb+wjuJK7XO</latexit> 0 1 Proof sketch <latexit sha1_base64="aaWHJn5I3Smwh1s+JpcIMsIm12Q=">AAACZXicfZDRShtBFIYnW6s2Vo21FEovHBoKtpSwW0RFEAR70RupglHBDeHs5GwcnJldZs5Wwxp8Gm/1eXyCvkZnYwo1Fg8MfPznH845f5Ir6SgM72vBi6mX0zOzr+pzr+cXFhtLb45cVliBbZGpzJ4k4FBJg22SpPAktwg6UXicnO9W/eNfaJ3MzCENcuxo6BuZSgHkpW7jvV2lz3xrm8dXsQY6S9LyYuil+KrbaIatcFT8KURjaLJx7XeXamtxLxOFRkNCgXOnUZhTpwRLUigc1uPCYQ7iHPp46tGARtcpRzcM+Sev9HiaWf8M8ZH6748StHMDnXhntaeb7FXif3uJnphM6WanlCYvCI14GJwWilPGq4B4T1oUpAYeQFjpd+fiDCwI8jHW6/F39MdZ3PODfuZogTL7pYzB9rU0Q39sP/5a0XNGuPxr9ORzjiZTfQpH31rRemv9YK25sztOfJZ9YB/ZKovYBtthP9g+azPBrtkNu2V3td/BfPA2ePdgDWrjP8vsUQUrfwBV5Lm+</latexit>
  76. r(t) := kw(t)k <latexit sha1_base64="lU9ZeMlw0wa94q+gsookqXGcyl0=">AAACcXicfVBdaxNBFJ2srdZYNbVP0pehodBWSXelVCkIgfTBF2kF0wS6Idyd3E2HzswuM3fVsOQX+Gt81V/i7+gf6Gw+QFPxwjCHc8/l3HuSXElHYfi7FjxYW3/4aONx/cnm02fPG1svLl1WWIFdkanM9hNwqKTBLklS2M8tgk4U9pKbTtXvfUHrZGY+0yTHgYaxkakUQJ4aNvZiDXSdpGUx3acDfvqeL4mvM+KIW/8NG82wFc6K3wfRAjTZoi6GW7XjeJSJQqMhocC5qyjMaVCCJSkUTutx4TAHcQNjvPLQgEY3KGf3TPmeZ0Y8zax/hviM/XOiBO3cRCdeWS3rVnsV+c9eolecKX03KKXJC0Ij5sZpoThlvAqLj6RFQWriAQgr/e5cXIMFQT7Sej0+Q3+cxY/e6DxHC5TZwzIGO9bSTP2x4/h1hf4nhG9LoUc+52g11fvg8k0rOmmdfDputjuLxDfYDttl+yxib1mbfWAXrMsE+85+sJ/sV+02eBnwYHcuDWqLmW32VwWv7gDIbL31</latexit> u(t) := w(t)/r(t) む Tangential

    + radial decomposition: w(t) = r(t) · u(t) <latexit sha1_base64="4J3K3RxwZLmE1hyw8v9g0Ed0VjA=">AAACdHicfVBdaxNBFJ2sXzVVm+qjPgwNgbRI2JVS+yIU6oMvYgTTFrIh3J3cTYfOxzJz1xqW/IX+mr7q//CP+OxskoKmpReGOZxzLvfekxVKeorj343owcNHj59sPG1uPnv+Yqu1/fLE29IJHAirrDvLwKOSBgckSeFZ4RB0pvA0uziu9dPv6Ly05hvNChxpmBqZSwEUqHGrm2qg8yyvLudd2uUfuKu/VEws8RuprKVxqx334kXx2yBZgTZbVX+83dhPJ1aUGg0JBd4Pk7igUQWOpFA4b6alxwLEBUxxGKABjX5ULU6a805gJjy3LjxDfMH+21GB9n6ms+Cst/TrWk3eqWV6bTLlh6NKmqIkNGI5OC8VJ8vrvPhEOhSkZgGAcDLszsU5OBAUUm02048YjnP4OQz6UqADsm6vSsFNtTTzcOw0fVuj+4zw48YYUMg5WU/1Njh510sOegdf99tHx6vEN9hrtsO6LGHv2RH7xPpswAS7YtfsJ/vV+BO9idpRZ2mNGqueV+y/inp/AW6ov7A=</latexit> む Derive ODE of tangential error ⇠ 1/t log t <latexit sha1_base64="+KQO3bCW4mewCdd+OjKkYLp5Wec=">AAACWHicfZDbShxBEIZrJwd1Y+Ihl7lpsgSChM2MiHop6EVuRIWsCs5Ganprx8Y+DN014jLse+TWvFV8GnvXDZg1pKDh46+/qaq/qLQKnKa/W8mLl69eLywutd8sv323srq2fhpc7SX1pNPOnxcYSCtLPVas6bzyhKbQdFZc70/6Zzfkg3L2O48q6hssrRoqiRylH3lQRmRfOdeuFHy52km76bTEc8hm0IFZHV+utbbygZO1IctSYwgXWVpxv0HPSmoat/M6UIXyGku6iGjRUOg307XH4lNUBmLofHyWxVR9+qNBE8LIFNFpkK/CfG8i/rNXmLnJPNztN8pWNZOVj4OHtRbsxCQTMVCeJOtRBJRexd2FvEKPkmNy7XZ+QPE4T4dx0FFFHtn5jSZHXxplx/HYMv8yof8Z8faPMVLMOZtP9Tmcbnaz7e72yVZnb3+W+CJ8gI/wGTLYgT34BsfQAwkefsId/GrdJ5AsJEuP1qQ1+/Me/qpk/QGp6bVK</latexit> ? む Control hZ↵, Pu u⇤ i at early stage <latexit sha1_base64="i6WJC5NFnLke0SgTAnGIXphVIEw=">AAACjnicfVBdaxQxFM2OX3X92uqjL8FFkFKWmVJWX0pL60NfxBXcttisw53sndnQTBKSjLgM83/6a3y1/TdmtiPoVrwQcnLuudyckxkpnI/j61505+69+w82HvYfPX7y9Nlg8/mJ05XlOOVaanuWgUMpFE698BLPjEUoM4mn2cVR2z/9htYJrT77pcFZCYUSueDgA5UODpkEVUikX1im5dwty3AxkGYB23SS1qwEv8hyWjVfmUFrunddNekWs6vRdDCMR/Gq6G2QdGBIupqkm71dNte8KlF5LsG58yQ2flaD9YJLbPqscmiAX0CB5wEqKNHN6pXZhr4OzJzm2oajPF2xf07UULrWRlC2f3XrvZb8Zy8r1zb7/N2sFspUHhW/WZxXknpN2yTpXFjkXi4DAG5F+DvlC7DAfci732fvMZiz+CEs+hiiA6/tVs3AFqVQTTBbsO0W/U8I338LAwo5J+up3gYnO6NkPBp/2h0eHHWJb5CX5BV5QxLylhyQYzIhU8LJJflBfpKraBCNo71o/0Ya9bqZF+Svio5/AWgEy1c=</latexit> Z↵ <latexit sha1_base64="ntsWT+hokTUIU4cKTIGYy8Kt/zE=">AAACXnicfVBdSxtBFJ1s1WqqNdoXwZfBUChSwm6QNHkT7IMvRQVjQt0Q7k5ukiHzsczMFsOSv+Kr/Ut986d0NiagQb0wzOGcc7n3niQV3Low/FcKPqytb3zc3Cp/2t75vFvZ27+xOjMM20wLbboJWBRcYdtxJ7CbGgSZCOwkk7NC7/xBY7lW126aYk/CSPEhZ+A81a/s/44TLQZ2Kv0Xg0jH0K9Uw1roq9GgBYiaYeRBq9Ws11s0mkthWCWLuuzvlU7igWaZROWYAGtvozB1vRyM40zgrBxnFlNgExjhrYcKJNpePl9+Rr96ZkCH2vinHJ2zzztykLZYzzsluLFd1QryVS2RK5PdsNnLuUozh4o9DR5mgjpNi2TogBtkTkw9AGa4352yMRhgzudXLsc/0R9n8JcfdJGiAafNcR6DGUmuZv7YUfy9QO8Z4W5p9MjnvAyTvg1u6rWoUWtcnVRPzxaJb5JDckS+kYj8IKfknFySNmHkjtyTB/K39BhsBDvB7pM1KC16vpAXFRz8BxpnuGQ=</latexit> u <latexit sha1_base64="61qfHGqKCNmsdzYd3BnJfVj0xwo=">AAACVHicfVBNS8NAEN3E7/qtRy+LRRCRkhSp7U3QgxdRwapoiky2k7q4uwm7G7GE/guv+rME/4sHN7WCijow8HjvDTPz4kxwY4Pg1fPHxicmp6ZnKrNz8wuLS8sr5ybNNcM2S0WqL2MwKLjCtuVW4GWmEWQs8CK+2y/1i3vUhqfqzPYz7EjoKZ5wBtZRV5EEexsnRT64WaoGtcBVo0FLEDaD0IFWq1mvt2g4lIKgSkZ1crPs7UTdlOUSlWUCjLkOg8x2CtCWM4GDSpQbzIDdQQ+vHVQg0XSK4ckDuuGYLk1S7VpZOmS/ThQgjenL2DnLE81PrSR/1WL5Y7NNmp2Cqyy3qNjH4iQX1Ka0zIN2uUZmRd8BYJq72ym7BQ3MutQqlegA3XMaj9yi4ww12FRvFRHonuRq4J7tRdsl+s8ID59Gh1zOn2HSv8F5vRY2ao3Tnere/ijxabJG1skmCcku2SOH5IS0CSOKPJIn8uy9eG/+mD/xYfW90cwq+Vb+wjuJK7XO</latexit> ? Pu u ⇤ 3 <latexit sha1_base64="0Fu26QSblnPQfllr+h6sSpOsW90=">AAACa3icfVDRShtBFJ1srdpYNda3KjgYBBEJu0Fi8ibYh74UIxgV3HS5O7kbB2dml5nZ0rDsY7+mr/ox/Qj/wdkYwYp6YeDMOedy7z1xJrixvv+v5n2Y+zi/sPipvvR5eWW1sfbl3KS5ZjhgqUj1ZQwGBVc4sNwKvMw0gowFXsQ3x5V+8Qu14ak6s5MMhxLGiiecgXVU1NjqR0UowV7HCc3Ln2GGOpv9i7yM9qJG02/5rjodWoGg6wcO9HrddrtHg6nk+00yq360VjsIRynLJSrLBBhzFfiZHRagLWcCy3qYG8yA3cAYrxxUINEMi+klJd1xzIgmqXZPWTpln3cUII2ZyNg5qyXNS60iX9Vi+WKyTbrDgqsst6jY4+AkF9SmtIqJjrhGZsXEAWCau90puwYNzLow6/XwG7rjNP5wg05cZmBTvVeEoMeSq9IdOw73K/SeEX4/GR1yOT+FSd8G5+1W0Gl1Tg+aR8ezxBfJBtkmuyQgh+SIfCd9MiCM/CF/yS25q917695Xb/PR6tVmPevkv/J2HgDhYb3W</latexit> 0 1 Proof sketch <latexit sha1_base64="aaWHJn5I3Smwh1s+JpcIMsIm12Q=">AAACZXicfZDRShtBFIYnW6s2Vo21FEovHBoKtpSwW0RFEAR70RupglHBDeHs5GwcnJldZs5Wwxp8Gm/1eXyCvkZnYwo1Fg8MfPznH845f5Ir6SgM72vBi6mX0zOzr+pzr+cXFhtLb45cVliBbZGpzJ4k4FBJg22SpPAktwg6UXicnO9W/eNfaJ3MzCENcuxo6BuZSgHkpW7jvV2lz3xrm8dXsQY6S9LyYuil+KrbaIatcFT8KURjaLJx7XeXamtxLxOFRkNCgXOnUZhTpwRLUigc1uPCYQ7iHPp46tGARtcpRzcM+Sev9HiaWf8M8ZH6748StHMDnXhntaeb7FXif3uJnphM6WanlCYvCI14GJwWilPGq4B4T1oUpAYeQFjpd+fiDCwI8jHW6/F39MdZ3PODfuZogTL7pYzB9rU0Q39sP/5a0XNGuPxr9ORzjiZTfQpH31rRemv9YK25sztOfJZ9YB/ZKovYBtthP9g+azPBrtkNu2V3td/BfPA2ePdgDWrjP8vsUQUrfwBV5Lm+</latexit>
  77. O(1/T ) む <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt L <latexit

    sha1_base64="i3VygpevB4bzpZuOYo4Bmh7n160=">AAACS3icfZDLSgMxFIYz1Xqpt6pLN4NFEJEyI1JdCrpwodiCvUCnlDPpaQ0mmSHJiGXoE7jVx/IBfA534sL0ImgrHgh8/OcP55w/jDnTxvPenMzcfHZhcWk5t7K6tr6R39yq6ShRFKs04pFqhKCRM4lVwwzHRqwQRMixHt6fD/v1B1SaRfLW9GNsCehJ1mUUjJUqV+18wSt6o3JnwZ9AgUyq3N50joNORBOB0lAOWjd9LzatFJRhlOMgFyQaY6D30MOmRQkCdSsdbTpw96zScbuRsk8ad6T+/JGC0LovQusUYO70dG8o/tkLxdRk0z1tpUzGiUFJx4O7CXdN5A5jcDtMITW8bwGoYnZ3l96BAmpsWLlccIH2OIXXdtBNjApMpA7SAFRPMDmwx/aCwyH9Z4THb6Mlm7M/neos1I6KfqlYqhwXzs4niS+RHbJL9olPTsgZuSRlUiWUIHkiz+TFeXXenQ/nc2zNOJM/2+RXZbJff+KzRg==</latexit> ⌘rL(wt ) <latexit sha1_base64="+IhGDSYic6dqVXm4SSEPluVOFR8=">AAACz3icfVFdSxtBFJ3dfmn6YWxfLH0ZGgpabdgNon0U7EMfWqrQqOCk4e7k7jo4M7vMzBrDGhH61J/Yn9B/0dkktTVKLwwczjmXe++ZpJDCuij6GYT37j94+GhhsfH4ydNnS83l5wc2Lw3HLs9lbo4SsCiFxq4TTuJRYRBUIvEwOd2t9cMzNFbk+qsbFdhTkGmRCg7OU/3m90+rTIE7SVJ6tkaZRPqXGK7RdSZBZ55lGhIJN8QNet1J313jITPTjnXKUgOcJeigwy7u9l70O986/WYrakeTordBPAMtMqu9/nKwyQY5LxVqxyVYexxHhetVYJzgEscNVlosgJ9ChscealBoe9UkrTF945kBTXPjn3Z0wv7bUYGydqQS76z3tPNaTd6pJWpuskvf9yqhi9Kh5tPBaSmpy2n9FXQgDHInRx4AN8LvTvkJ+Myc/7BGg31Af5zBz37QlwINuNy8rRiYTAk99sdmbKNG/zPC+R+jRz7neD7V2+Cg04632lv7m62d3VniC+QVeU1WSUy2yQ75SPZIl3DyK1gKVoKX4X44DC/Dq6k1DGY9L8iNCn/8Boen3Vc=</latexit> L(v)  L(w) + hrL(w), v む wi + 2 kv 2 wk2 ⟺ C - 2 <latexit sha1_base64="qSSMdMjLzVym3WAWE13PVeLyf6E=">AAACk3icfVFdSxtBFJ1sbavph9Hiky9DQyGWEnZFrJAXqYI+KFpoVHDScGdyNw7OzC4zs23Dkp/kr/GpUP+Ls3GhbSy9MHA491zOvWd4rqTzcfyzET1ZePrs+eJS88XLV6+XWyurZy4rrMC+yFRmLzg4VNJg30uv8CK3CJorPOfXe1X//BtaJzPzxU9yHGgYG5lKAT5Qw9YBU0E8gmHJNPyYdpgBruDrJj3qBMJf8ZR+39igTCFlHD1Q1mO9DkszC0rR35Jhqx1341nRxyCpQZvUdTpcaWyxUSYKjcYLBc5dJnHuByVYL4XCaZMVDnMQ1zDGywANaHSDcnbxlL4LzIiGLcIzns7YPydK0M5NNA/KakU336vIf/a4nnP26c6glCYvPBrxYJwWivqMVnHSkbQovJoEAMLKsDsVV2BB+BB6s8n2MRxn8TgYneRowWf2fcnAjrU003DsmH2o0P+E4VtqYUAh52Q+1cfgbLObbHe3P2+1d/fqxBfJOnlLOiQhH8kuOSSnpE8EuSG35Be5i9aiXvQp2n+QRo165g35q6Ljey0vymM=</latexit> max (r L(w))  ⌘  2/ <latexit sha1_base64="wzHh3HCDHQEVQunkPk/EINhiauc=">AAACXHicfZBNSwMxEIbT9bt+VQUvXoJFEJG6K6IeBT14ERWsCm4ps+m0BpPsksyKZek/8ar/yYu/xbRW0CoOBB7eecPMvEmmpKMwfCsFY+MTk1PTM+XZufmFxcrS8rVLcyuwLlKV2tsEHCppsE6SFN5mFkEnCm+Sh+N+/+YRrZOpuaJuhg0NHSPbUgB5qVmpxEjAY4V8dydOPDcr1bAWDor/hmgIVTasi+ZSaS9upSLXaEgocO4uCjNqFGBJCoW9cpw7zEA8QAfvPBrQ6BrFYPUe3/BKi7dT658hPlC//yhAO9fViXdqoHs32uuLf/YSPTKZ2oeNQposJzTic3A7V5xS3s+Ft6RFQarrAYSVfncu7sGCIJ9euRyfoD/O4pkfdJ6hBUrtVhGD7Whpev7YTrzdp/+M8PRl9ORzjkZT/Q3Xu7Vov7Z/uVc9Oh4mPs3W2DrbZBE7YEfslF2wOhPskT2zF/Zaeg/Gg9lg/tMalIZ/VtiPClY/AJVPtiA=</latexit> : descent lemma む <latexit sha1_base64="JtSbEOdrK83T9gPDi3ZI0tyc2Go=">AAADd3icfVLLbhMxFPUkPMrwaAtLFlhEREnTRJmoatkgVSoLFiCKRNpKdYjuOHcmVj2eke0Uokk+gq9hC5/Bp7DDkwSpnZZcydbRPffclx1mUhjb7f72KtU7d+/d33jgP3z0+Mnm1vbTE5NONMc+T2Wqz0IwKIXCvhVW4lmmEZJQ4ml4cVTwp5eojUjVZzvNcJBArEQkOFjnGm57O+8bLAE7DqP863yY21Ywb/r1N/Sa29I2ZWiBKQgllLhmkzLm15nEMkFbTIKKHXG7cJe2/5+VMr3UtiiLNHAWutAem62RsNmw96W36KY8QLNIIzGyjavJ3NUOmBbx2ImLtLN1iYdbtW6nuzB6EwQrUCMrO3bb3WOjlE8SVJZLMOY86GZ2kIO2gkuc+2xiMAN+ATGeO6ggQTPIF686p6+cZ0SjVLujLF14rypySIyZJqGLLHo1Za5w3sqFSamyjV4PcqGyiUXFl4WjiaQ2pcWXoSOhkVs5dQC4Fq53ysfgtmjdx/J99hbdcBo/uEIfM9RgU72TM9BxItTcDRuz3QKtC4Rv/wIdcnsOylu9CU56nWC/s/9pr3Z4tNr4BnlOXpIGCcgBOSTvyDHpE+599354P71flT/VF9V6tbEMrXgrzTNyzarBX99wGhk=</latexit> L(wt+1 ) = L(wt <latexit sha1_base64="5+hr1bFxR7zFk+0KxrTyGisgJ9U=">AAADjHichVJdaxNBFJ3t+lFX7ZePvgwGS2LbkA2hFrRYqIgPihVMW+jEcHdydzN0dnaZmahhk7/jr/FVwX/jbBKh3dZ6YeFwzzn3ayfKpTC21frtLfm3bt+5u3wvuP/g4crq2vrGsclGmmOXZzLTpxEYlEJh1wor8TTXCGkk8SQ6Pyz5ky+ojcjUJzvOsZdCokQsOFiX6q97r97VWQp2GMXF12m/sFvhtBFs7tNLaUt3KEMLTEEkocI1GpSxYJNJrBJ0h0lQiSOuN247wT+rUqbn3i3KYg2cRU7aZpMbLGzSb39uz6apLtAoy0iMbf1CsXkpLZKh85Zw8t+6L6tcf63WarZmQa+CcAFqZBFH7uQdNsj4KEVluQRjzsJWbnsFaCu4xGnARgZz4OeQ4JmDClI0vWL2q6f0qcsMaJxp9ylLZ9mLjgJSY8Zp5JTllKbKlclruSitdLbxXq8QKh9ZVHzeOB5JajNaviM6EBq5lWMHgGvhZqd8CO601r22IGCv0S2n8b1r9CFHDTbTzwoGOkmFmrplE7ZdopuE8O2v0CF357B61avguN0Md5u7Hzu1g8PFxZfJY/KE1ElInpMD8pYckS7h3nfvh/fT++Wv+B3/hb8/ly55C88jcin8N38A0Kcgzg==</latexit> <latexit sha1_base64="JtSbEOdrK83T9gPDi3ZI0tyc2Go=">AAADd3icfVLLbhMxFPUkPMrwaAtLFlhEREnTRJmoatkgVSoLFiCKRNpKdYjuOHcmVj2eke0Uokk+gq9hC5/Bp7DDkwSpnZZcydbRPffclx1mUhjb7f72KtU7d+/d33jgP3z0+Mnm1vbTE5NONMc+T2Wqz0IwKIXCvhVW4lmmEZJQ4ml4cVTwp5eojUjVZzvNcJBArEQkOFjnGm57O+8bLAE7DqP863yY21Ywb/r1N/Sa29I2ZWiBKQgllLhmkzLm15nEMkFbTIKKHXG7cJe2/5+VMr3UtiiLNHAWutAem62RsNmw96W36KY8QLNIIzGyjavJ3NUOmBbx2ImLtLN1iYdbtW6nuzB6EwQrUCMrO3bb3WOjlE8SVJZLMOY86GZ2kIO2gkuc+2xiMAN+ATGeO6ggQTPIF686p6+cZ0SjVLujLF14rypySIyZJqGLLHo1Za5w3sqFSamyjV4PcqGyiUXFl4WjiaQ2pcWXoSOhkVs5dQC4Fq53ysfgtmjdx/J99hbdcBo/uEIfM9RgU72TM9BxItTcDRuz3QKtC4Rv/wIdcnsOylu9CU56nWC/s/9pr3Z4tNr4BnlOXpIGCcgBOSTvyDHpE+599354P71flT/VF9V6tbEMrXgrzTNyzarBX99wGhk=</latexit> ⌘rL(wt )) 十 2 5  L(wt ) + hrL(wt ), ⌘rL(wt )i + k 2 ✓ ◆◆ 22 2 ⌘ ⌘ 1 ⌘krL(w ⌘krL(wt )k )k = L(wt ) + tt 2 2 2 2 < L(wt ) 0 1 <latexit sha1_base64="idkAoaQdalqMhq1DnbbnuR1Z6y4=">AAACUHicfVBNTxsxEJ1NP6ALbaEce7GIKkFVbb0kDTki0QMXBJVIQGIjNOtMgovtXdneqtEq/6FX+rN66z/prTiBSBS1Hcny05s3ejMvL5V0nvOfUePR4ydPl5afxSurz1+8XFt/1XdFZQX1RKEKe5ajIyUN9bz0is5KS6hzRaf51f6sf/qFrJOFOfGTkgYax0aOpEAfqP7RVvr+ZPtirckT3uq0Wm3Gkxbv8jnY5bz7IWVpwufVhLs6vliP2tmwEJUm44VC585TXvpBjdZLoWgaZ5WjEsUVjuk8QIOa3KCerztlbwIzZKPChmc8m7P3J2rUzk10HpQa/aV72JuRf+3l+oGzH3UHtTRl5cmIW+NRpZgv2CwLNpSWhFeTAFBYGXZn4hItCh8Si+PsI4XjLB0Go6OSLPrCvq0ztGMtzTQcO87ezdD/hPh1IQwo5LwIk/0b9HeStJN0PrWbe/t3iS/Da9iELUhhF/bgAI6hBwI+wze4hu/Rj+hX9LsR3UoXP2zAH9WIbwB9lbO7</latexit> 2 ⌘rL(wt )k2 smoothness 2 ⌘  2/ <latexit sha1_base64="d4Uyi6fmASIkHcAbdWOXfDlRXAM=">AAACWHicfZDfSxtBEMcn1x9qamtMH305GgoiJb0TiT4K8aEvpQpGA7koc5tJXNzdO3bnxHDk/+ir/a/av6abeIJGcWDhw3e+y8x801xJx1H0txa8efvu/crqWv3D+sdPG43N5pnLCiuoJzKV2X6KjpQ01GPJivq5JdSpovP0ujvvn9+QdTIzpzzNaahxYuRYCmQvXSTEmCja/Z6kni4bragdLSp8DnEFLajq+HKztpeMMlFoMiwUOjeIo5yHJVqWQtGsnhSOchTXOKGBR4Oa3LBcrD0Lv3plFI4z65/hcKE+/lGidm6qU+/UyFduuTcXX+ylemkyjw+GpTR5wWTE/eBxoULOwnkm4UhaEqymHlBY6XcPxRVaFOyTq9eTI/LHWfrpB/3KySJndqdM0E60NDN/7CT5NqfXjHj7YPTkc46XU30OZ7vtuNPunOy1DrtV4quwBV9gG2LYh0P4AcfQAwEWfsMd/Kn9CyBYCdburUGt+vMZnlTQ/A9GUrWb</latexit> (8w)
  78. [Li+ ] 大 8 1 0 2 6 Li, H.,

    Xu, Z., Taylor, G., Studer, C., & Goldstein, T. (2018). Visualizing the loss landscape of neural nets. 0 1 📄 む
  79. <latexit sha1_base64="Ex2Zxj2vzLIdVQDIXvIE9YDkT+g=">AAACkHicfVDbahRBEO0db3G9beKjL42LkIgsMyHEIIiR5EFEMaKbBDLrUtNbs2nS3TN012iWYX/Ir/E1fo01m3mIG7Gg4XDOqa6qk5VGB4rji0504+at23dW7nbv3X/w8FFvde0wFJVXOFSFKfxxBgGNdjgkTQaPS49gM4NH2dleox99Rx904b7SrMSRhanTuVZATI17+ynhOdVfTsGXDkOYr6cW6DTL5Y8N+eq1TA3/NYFxzfQ5iw4yA9825Ycrvo1xrx8P4kXJ6yBpQV+0dTBe7Wylk0JVFh0pAyGcJHFJoxo8aWVw3k2rgCWoM5jiCUMHFsOoXpw7l8+Ymci88PwcyQV7taMGG8LMZuxsdgzLWkP+U8vs0mTKd0a1dmVF6NTl4LwykgrZZCkn2qMiM2MAymveXSoOEhRx4t1uuo98nMePPOhTiR6o8M/rFPzUajfnY6fpiwb9z8iht0ZGnHOynOp1cLg5SLYH25+3+rt7beIr4ol4KtZFIl6KXfFOHIihUOKn+CUuxO9oLdqJ3kRvL61Rp+15LP6q6P0fVg/KHQ==</latexit> Sharpness(w) := 7 色 cf. 0 1 大

    (Edge of stability) <latexit sha1_base64="5FSkUlokt7wtfo9S2WxLMEMRtnw=">AAACinicfZDPahRBEMZ7J1HjGnWjx1waFyFKWGZCiIoogeTgRYzoJoH0stT01mya9D+6e4zLMG/j03iNl7xNeiYb0Y1Y0PSPqq+oqi+3UviQppedZGn5zt17K/e7D1YfPnrcW3ty6E3pOA65kcYd5+BRCo3DIILEY+sQVC7xKD/ba+pH39B5YfTXMLM4UjDVohAcQkyNe+9ZjgHoO8p8accVUxBO84Ke12yzZWPbz6nqyyk4q9H7euO36sW4108HaRv0NmRz6JN5HIzXOttsYnipUAcuwfuTLLVhVIELgkusu6z0aIGfwRRPImpQ6EdVe2hNn8fMhBbGxacDbbN/dlSgvJ+pPCqbFf1irUn+s5arhcmheD2qhLZlQM2vBxelpMHQxkU6EQ55kLMIwJ2Iu1Me/QEeotfdLtvHeJzDj3HQJ4sOgnEvKwZuqoSu47FTttnQ/4Tw/UYYKfqcLbp6Gw63BtnOYOfzdn93b+74Clknz8gGycgrsks+kAMyJJz8ID/JBfmVrCZbyZvk7bU06cx7npK/Itm/ApCIyS0=</latexit> = sup Sharpness(w) w 2 max (r L(w))
  80. <latexit sha1_base64="Ex2Zxj2vzLIdVQDIXvIE9YDkT+g=">AAACkHicfVDbahRBEO0db3G9beKjL42LkIgsMyHEIIiR5EFEMaKbBDLrUtNbs2nS3TN012iWYX/Ir/E1fo01m3mIG7Gg4XDOqa6qk5VGB4rji0504+at23dW7nbv3X/w8FFvde0wFJVXOFSFKfxxBgGNdjgkTQaPS49gM4NH2dleox99Rx904b7SrMSRhanTuVZATI17+ynhOdVfTsGXDkOYr6cW6DTL5Y8N+eq1TA3/NYFxzfQ5iw4yA9825Ycrvo1xrx8P4kXJ6yBpQV+0dTBe7Wylk0JVFh0pAyGcJHFJoxo8aWVw3k2rgCWoM5jiCUMHFsOoXpw7l8+Ymci88PwcyQV7taMGG8LMZuxsdgzLWkP+U8vs0mTKd0a1dmVF6NTl4LwykgrZZCkn2qMiM2MAymveXSoOEhRx4t1uuo98nMePPOhTiR6o8M/rFPzUajfnY6fpiwb9z8iht0ZGnHOynOp1cLg5SLYH25+3+rt7beIr4ol4KtZFIl6KXfFOHIihUOKn+CUuxO9oLdqJ3kRvL61Rp+15LP6q6P0fVg/KHQ==</latexit> Sharpness(w) := 8 色 cf. 0 1 大

    (Edge of stability) <latexit sha1_base64="5FSkUlokt7wtfo9S2WxLMEMRtnw=">AAACinicfZDPahRBEMZ7J1HjGnWjx1waFyFKWGZCiIoogeTgRYzoJoH0stT01mya9D+6e4zLMG/j03iNl7xNeiYb0Y1Y0PSPqq+oqi+3UviQppedZGn5zt17K/e7D1YfPnrcW3ty6E3pOA65kcYd5+BRCo3DIILEY+sQVC7xKD/ba+pH39B5YfTXMLM4UjDVohAcQkyNe+9ZjgHoO8p8accVUxBO84Ke12yzZWPbz6nqyyk4q9H7euO36sW4108HaRv0NmRz6JN5HIzXOttsYnipUAcuwfuTLLVhVIELgkusu6z0aIGfwRRPImpQ6EdVe2hNn8fMhBbGxacDbbN/dlSgvJ+pPCqbFf1irUn+s5arhcmheD2qhLZlQM2vBxelpMHQxkU6EQ55kLMIwJ2Iu1Me/QEeotfdLtvHeJzDj3HQJ4sOgnEvKwZuqoSu47FTttnQ/4Tw/UYYKfqcLbp6Gw63BtnOYOfzdn93b+74Clknz8gGycgrsks+kAMyJJz8ID/JBfmVrCZbyZvk7bU06cx7npK/Itm/ApCIyS0=</latexit> = sup Sharpness(w) w 2 max (r L(w)) stay on edge e v i s g s n e i r n g e o p r r P a sh <latexit sha1_base64="FOpfjo1PL0jpMKhD7AEsUswjgyo=">AAACY3icfVBdaxNBFJ2sH62ptWkVfBBhMAhSJGy2Malvhfrgi1jBtIVuCHcnd9Oh88XMXWlY8mt81R/kD/B/ONumoEU9MHA491zO3FM4JQOl6Y9Wcufuvftr6w/aGw83H211tneOg628wLGwyvrTAgIqaXBMkhSeOo+gC4UnxcVhMz/5gj5Iaz7TwuFEw9zIUgqgKE07T/MCCXJwztvLnEoPIsujMu10097b/WH2JuNpL01H2d6wIdlokO3xflQadNkKR9Pt1iCfWVFpNCQUhHDWTx1NavAkhcJlO68COhAXMMezSA1oDJP66oIlfxmVGS+tj88Qv1J/36hBh7DQRXRqoPNwe9aIf50V+lYylfuTWhpXERpxHVxWipPlTT18Jj0KUotIQHgZ/87FOcRWKJbYbufvMB7n8UMM+ujQA1m/W+fg51qaZTx2nr9u2P+McHljjCz2fFMm/zc5znr9YW/4adA9OFw1vs6esRfsFeuzETtg79kRGzPBluwr+8a+t34mG8lO8uTamrRWO4/ZH0ie/wKE+bpw</latexit> 2 ⇡⌘
  81. 1 2 0 2 9 む 0 1 (Edge of

    stability) [Cohen+ ]
  82. | Self-stabilization む Progressive sharpening h rL(w), rS(w)i > 0

    <latexit sha1_base64="J2fiRnkwBcSkCvDbK63V0ScrP1o=">AAACiXicfZDRShtBFIYn21ZtbG1sL70ZDIItNuwWUSkoQrzwQqlio4ITwtnJ2XVwZnaZma2GJU/Tp/G2XvVtOptswcbigYGP//yHM+ePcymsC8PfjeDFy1dz8wuvm4tv3i69ay2/P7dZYTj2eCYzcxmDRSk09pxwEi9zg6BiiRfxTbfqX/xAY0Wmv7tRjn0FqRaJ4OC8NGjtMgk6lUg/Mw2xBHq0zhS46zihtx83aC2ePRIpM9OJPRoOWu2wE06KPoWohjap62Sw3Nhkw4wXCrXjEqy9isLc9UswTnCJ4yYrLObAbyDFK48aFNp+OblzTNe8MqRJZvzTjk7UxxMlKGtHKvbO6rt2tleJ/+3FamazS3b6pdB54VDz6eKkkNRltAqRDoVB7uTIA3Aj/N8pvwYD3Pmom012gP44g8d+0bccDbjMfCoZmFQJPfbHpmyjoueMcPfX6MnnHM2m+hTOv3Sirc7W6WZ7v1snvkBWyCpZJxHZJvvkkJyQHuHkJ7knv8hDsBhEwU7wdWoNGvXMB/JPBd0/QZbFkQ==</latexit> sharpness sharpness S(w) := max (r2 L(w)) <latexit sha1_base64="sV06HrOc18u6iGcP+5XUPDoXkwk=">AAACgXicfZDbSiNBEIY7o+shezCul940hoVkWcJM8ISLIOiFF4rKblRwsqGmU5M0dvcM3T27hiFP4tN4q0/g29gT50LjYkHDx19/UV1/lApurO8/VryZ2Q9z8wuL1Y+fPn9Zqi1/PTdJphl2WCISfRmBQcEVdiy3Ai9TjSAjgRfR9X7Rv/iL2vBE/bajFLsSBorHnIF1Uq+28asRSrDDKKb/mnRnl4bCDfehlzv5ZtwIFUQC/rTp0Qtfs1er+y1/UvQtBCXUSVmnveXKethPWCZRWSbAmKvAT203B205EziuhpnBFNg1DPDKoQKJpptP7hvTb07p0zjR7ilLJ+rLiRykMSMZOWfxRzPdK8T/9iI5tdnG292cqzSzqNjz4jgT1Ca0CI/2uUZmxcgBMM3d3ykbggZmXcTVaniA7jiNx27RSYoabKK/5yHogeRq7I4dhD8Kes/oQi+NjlzOwXSqb+G83Qo2W5tn6/W9/TLxBbJK1kiDBGSL7JFDcko6hJFbckfuyYM34zU932s/W71KObNCXpX38wlB18MK</latexit> む sharpness sharpness 3 2 Alex Damian 立 方 用 氏 0 https://alex-damian.github.io/ 1 1 🔗 EoS Web
  83. | Self-stabilization [Damian+ む Progressive sharpening h rL(w), rS(w)i >

    0 ] <latexit sha1_base64="J2fiRnkwBcSkCvDbK63V0ScrP1o=">AAACiXicfZDRShtBFIYn21ZtbG1sL70ZDIItNuwWUSkoQrzwQqlio4ITwtnJ2XVwZnaZma2GJU/Tp/G2XvVtOptswcbigYGP//yHM+ePcymsC8PfjeDFy1dz8wuvm4tv3i69ay2/P7dZYTj2eCYzcxmDRSk09pxwEi9zg6BiiRfxTbfqX/xAY0Wmv7tRjn0FqRaJ4OC8NGjtMgk6lUg/Mw2xBHq0zhS46zihtx83aC2ePRIpM9OJPRoOWu2wE06KPoWohjap62Sw3Nhkw4wXCrXjEqy9isLc9UswTnCJ4yYrLObAbyDFK48aFNp+OblzTNe8MqRJZvzTjk7UxxMlKGtHKvbO6rt2tleJ/+3FamazS3b6pdB54VDz6eKkkNRltAqRDoVB7uTIA3Aj/N8pvwYD3Pmom012gP44g8d+0bccDbjMfCoZmFQJPfbHpmyjoueMcPfX6MnnHM2m+hTOv3Sirc7W6WZ7v1snvkBWyCpZJxHZJvvkkJyQHuHkJ7knv8hDsBhEwU7wdWoNGvXMB/JPBd0/QZbFkQ==</latexit> sharpness sharpness S(w) := max (r2 L(w)) yt <latexit sha1_base64="sV06HrOc18u6iGcP+5XUPDoXkwk=">AAACgXicfZDbSiNBEIY7o+shezCul940hoVkWcJM8ISLIOiFF4rKblRwsqGmU5M0dvcM3T27hiFP4tN4q0/g29gT50LjYkHDx19/UV1/lApurO8/VryZ2Q9z8wuL1Y+fPn9Zqi1/PTdJphl2WCISfRmBQcEVdiy3Ai9TjSAjgRfR9X7Rv/iL2vBE/bajFLsSBorHnIF1Uq+28asRSrDDKKb/mnRnl4bCDfehlzv5ZtwIFUQC/rTp0Qtfs1er+y1/UvQtBCXUSVmnveXKethPWCZRWSbAmKvAT203B205EziuhpnBFNg1DPDKoQKJpptP7hvTb07p0zjR7ilLJ+rLiRykMSMZOWfxRzPdK8T/9iI5tdnG292cqzSzqNjz4jgT1Ca0CI/2uUZmxcgBMM3d3ykbggZmXcTVaniA7jiNx27RSYoabKK/5yHogeRq7I4dhD8Kes/oQi+NjlzOwXSqb+G83Qo2W5tn6/W9/TLxBbJK1kiDBGSL7JFDcko6hJFbckfuyYM34zU932s/W71KObNCXpX38wlB18MK</latexit> <latexit sha1_base64="1//p2HkB9b8P3fSMRvIrUVAbbMU=">AAACTXicfZBNSwMxEIaz9bt+69HLYhFEpOyKqEdBD15ERauCW8psOl1Dk+ySzIpl6U/wqj/Lsz/Em4hpraBVHAg8vPOGmXnjTApLQfDilUZGx8YnJqfK0zOzc/MLi0uXNs0NxxpPZWquY7AohcYaCZJ4nRkEFUu8itsHvf7VHRorUn1BnQzrChItWoIDOem806DGQiWoBv3yf0M4gAob1Glj0duOminPFWriEqy9CYOM6gUYElxitxzlFjPgbUjwxqEGhbZe9Hft+mtOafqt1Linye+r338UoKztqNg5FdCtHe71xD97sRqaTK29eiF0lhNq/jm4lUufUr8XhN8UBjnJjgPgRrjdfX4LBji5uMrl6BDdcQaP3aCTDA1QajaKCEyihO66Y5Nos0f/GeH+y+jI5RwOp/obLreq4U5152y7sn8wSHySrbBVts5Ctsv22RE7ZTXGWcIe2CN78p69V+/Ne/+0lrzBn2X2o0oTH7LhtFo=</latexit> 2 む u loss Hessian rS sharpness <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> ( Instability xt := hu, wt u <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> xt r2 L(w)u = S(w)u ) <latexit sha1_base64="OLiKG9AWP6m0Od3isOVdG94OD4M=">AAACTXicfZBNSwMxEIaz9bt+69HLYhFEpOyKqEdBD15ERauCW8psOl1Dk+ySzIpl6U/wqj/Lsz/Em4hpraBVHAg8vPOGmXnjTApLQfDilUZGx8YnJqfK0zOzc/MLi0uXNs0NxxpPZWquY7AohcYaCZJ4nRkEFUu8itsHvf7VHRorUn1BnQzrChItWoIDOen8vkGNhUpQDfrl/4ZwABU2qNPGorcdNVOeK9TEJVh7EwYZ1QswJLjEbjnKLWbA25DgjUMNCm296O/a9dec0vRbqXFPk99Xv/8oQFnbUbFzKqBbO9zriX/2YjU0mVp79ULoLCfU/HNwK5c+pX4vCL8pDHKSHQfAjXC7+/wWDHBycZXL0SG64wweu0EnGRqg1GwUEZhECd11xybRZo/+M8L9l9GRyzkcTvU3XG5Vw53qztl2Zf9gkPgkW2GrbJ2FbJftsyN2ymqMs4Q9sEf25D17r96b9/5pLXmDP8vsR5UmPgCw+LRZ</latexit> <latexit sha1_base64="v0kLZjyNWViiTn7XiSs5aKFEHm0=">AAACgXicfZDRahNBFIYnW21rtG1iL70ZDIVGJOyGNgpSCMQLLxQjNmkgG8PZydnt0JnZZWZWG5Y8SZ/GW30C38bZNAVNQw8MfPznH845f5QJbqzv/6l4W48eb+/sPqk+fba3f1CrPx+aNNcMBywVqR5FYFBwhQPLrcBRphFkJPAiuuqV/YvvqA1P1bmdZziRkCgecwbWSdPaaaggEvCt/fE4lGAvo5j+aN5RTs/o1036tNbwW/6y6H0IVtAgq+pP65WTcJayXKKyTIAx48DP7KQAbTkTuKiGucEM2BUkOHaoQKKZFMv7FvTIKTMap9o9ZelS/fdHAdKYuYycs9zQrPdKcWMvkmuTbfx2UnCV5RYVux0c54LalJbh0RnXyKyYOwCmududskvQwKyLuFoN36M7TuMnN+hzhhpsql8VIehEcrVwxybh65IeMsL1ndGRyzlYT/U+DNutoNPqfDlpdHurxHfJC/KSHJOAvCFd8oH0yYAwckN+kl/kt7flNT3fa99avcrqzyH5r7x3fwEL0cNv</latexit> Sharpening w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> yt := hrS, wt w0 i <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> && S(w0 ) = 2/⌘ 2 Alex Damian Web <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> r L(w0 ) t=0 stable hrL(w0 ), ui = 0 <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> 方 方 用 方 方 大 3 2 0 2 氏 1 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS
  84. | Self-stabilization [Damian+ yt = ⌘h rL(wt ), rSi '

    ⌘h rL(w0 ), rSi <latexit sha1_base64="Iqu+raqFrpnJXvLeoggt2V8Gudo=">AAACzHicfVFNaxRBEO0Zv+IazUbRi5fGRYiaLDMSohchkBw8+BHRTQLpZajprZ006e4Zu2vUZVjw5H/MH/B32LO7gm5CChoer96jql7nlVaekuQ8iq9dv3Hz1srtzp3Vu/fWuuv3D31ZO4kDWerSHefgUSuLA1Kk8bhyCCbXeJSf7bX9o2/ovCrtF5pUODRQWDVWEihQWffnJGvoRTrlW3ySEX/DBRIIDbbQyLeEhVwDf7chDNBpPubfM3q2yRf0Z+HmOuGVwa8zK7/Km1zizbq9pJ/Mil8E6QL02KIOsvVoW4xKWRu0JDV4f5ImFQ0bcKSkxmlH1B4rkGdQ4EmAFgz6YTOLasqfBmbEx6ULzxKfsf86GjDeT0welO3afrnXkpf2crM0mcavh42yVU1o5XzwuNacSt7+Ax8ph5L0JACQToXduTwFB5LCb3U6Yh/DcQ7fh0EfK3RApXveCHCFUXYaji3EZouuEsKPv8KAQs7pcqoXweHLfrrT3/m03dvdWyS+wh6zJ2yDpewV22Vv2QEbMMl+R6vRw+hR/CGmuImnc2kcLTwP2H8V//oDjOnckw==</latexit> yt+1 progressive sharpening (Taylor ] ) >0 sharpness <latexit sha1_base64="jF3VEmxBcTwjVE7nk8udiwvpT0o=">AAADAXicjVFNbxMxEPUuXyV8NIVjLxYRUlNKtBuqwrGoHBAfokikrdQN0awzm1q1vYs9C0SrnPg13BASJ34J/wZvukhtWhAjWXp6b55n/JwWSjqKol9BeOnylavXlq63bty8dXu5vXJnz+WlFTgQucrtQQoOlTQ4IEkKDwqLoFOF++nxTq3vf0TrZG7e0bTAoYaJkZkUQJ4atX8kBlIF/NVaooGO0ox/GlGXJ05q/MAvEKMuf9Dw7/tnhdNXPFx0ZBZE3G+Mj/7LuL7xF747aneiXjQvfh7EDeiwpnZHK8FmMs5FqdGQUODcYRwVNKzAkhQKZ62kdFiAOIYJHnpoQKMbVvN8Z/y+Z8Y8y60/hvicPe2oQDs31anvrJd0i1pNXqilemEyZU+GlTRFSWjEyeCsVJxyXn8eH0uLgtTUAxBW+t25OAKfLPkvbrWSZ+gfZ/G1H/SmQAuU2/UqATvR0sz8YyfJRo3+1Qif/zR65HOOF1M9D/b6vXirt/V2s7O90yS+xFbZPbbGYvaYbbPnbJcNmAhWg6fBi+Bl+CX8Gn4Lv5+0hkHjucvOVPjzN4pm70U=</latexit> 2 rL(wt ) ' rL(w0 ) + r L(w0 )(wt 1 3 w0 ) + r L(w0 )(wt 2 2 <latexit sha1_base64="8uSa4n0wdODiLbonR7l3nrwKuCk=">AAACaXicfZDbSiNBEIY7sx6zHpL1RpSFZoMQFwkzIq6Xgl54oRhZE4VMDDWdStKku2fo7lkMQ658Gm/1aXyGfYntHASNiwUNH3/9RXX9USK4sb7/kvO+zM0vLC4t57+urK6tF4rf6iZONcMai0WsbyMwKLjCmuVW4G2iEWQk8Cbqn4z6N39QGx6raztIsCmhq3iHM7BOahW+hwoiAfQ3DbmiYR91eaLc7dPz3Vah5Ff8cdGPEEyhRKZVbRVzB2E7ZqlEZZkAYxqBn9hmBtpyJnCYD1ODCbA+dLHhUIFE08zGdwzpjlPatBNr95SlY/XtRAbSmIGMnFOC7ZnZ3kj8by+SM5tt56iZcZWkFhWbLO6kgtqYjkKiba6RWTFwAExz93fKeqCBWRdlPh+eojtO44VbdJmgBhvrn1kIuiu5Grpju+HeiD4zwv2r0ZHLOZhN9SPU9yvBYeXw6qB0fDJNfIlskx+kTALyixyTM1IlNcLIA3kkT+Q599crepve1sTq5aYzG+RdeaV/y0W6yw==</latexit> rS 2 ker(r L) u 2 <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> r L(w0 ) && S(w0 ) = 2/⌘ <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> t=0 stable hrL(w0 ), ui = 0 <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> Stage w ⇤ , wt w t ' w0 <latexit sha1_base64="PCkDcUC+Zyxa1FPU8AMhgSzdCnY=">AAACanicfZDfShtBFMYnW60a+yfqlQgymAqllLBbxPZS0AtvShWMCm4IZydn4+DM7DpzthqW3Pk03urL+A4+hLMxoo3FAwM/vvMN55wvyZV0FIZ3teDd1PT7mdm5+vyHj58+NxYWD11WWIFtkanMHifgUEmDbZKk8Di3CDpReJScbVf9o79onczMAQ1y7GjoG5lKAeSlbmM11kCnScovusRjJzWe82cp7DaaYSscFX8N0RiabFx73YXaRtzLRKHRkFDg3EkU5tQpwZIUCof1uHCYgziDPp54NKDRdcrRIUO+7pUeTzPrnyE+Ul/+KEE7N9CJd1Y7usleJf63l+iJyZT+6pTS5AWhEY+D00JxyniVEu9Ji4LUwAMIK/3uXJyCBUE+y3o93kF/nMXfftCfHC1QZr+VMdi+lmboj+3H3yt6ywiXT0ZPPudoMtXXcPijFW22Nvc3mlvb48Rn2QpbY19ZxH6yLbbL9libCXbFrtkNu63dB4vBcrDyaA1q4z9L7J8KvjwAqXq8Nw==</latexit> Instability Sharpening 方 立 3 2 0 2 高 小 1 2 w⇤ ) : 0 xt := hu, wt w0 i yt := hrS, wt w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS
  85. | Self-stabilization [Damian+ <latexit sha1_base64="Iqu+raqFrpnJXvLeoggt2V8Gudo=">AAACzHicfVFNaxRBEO0Zv+IazUbRi5fGRYiaLDMSohchkBw8+BHRTQLpZajprZ006e4Zu2vUZVjw5H/MH/B32LO7gm5CChoer96jql7nlVaekuQ8iq9dv3Hz1srtzp3Vu/fWuuv3D31ZO4kDWerSHefgUSuLA1Kk8bhyCCbXeJSf7bX9o2/ovCrtF5pUODRQWDVWEihQWffnJGvoRTrlW3ySEX/DBRIIDbbQyLeEhVwDf7chDNBpPubfM3q2yRf0Z+HmOuGVwa8zK7/Km1zizbq9pJ/Mil8E6QL02KIOsvVoW4xKWRu0JDV4f5ImFQ0bcKSkxmlH1B4rkGdQ4EmAFgz6YTOLasqfBmbEx6ULzxKfsf86GjDeT0welO3afrnXkpf2crM0mcavh42yVU1o5XzwuNacSt7+Ax8ph5L0JACQToXduTwFB5LCb3U6Yh/DcQ7fh0EfK3RApXveCHCFUXYaji3EZouuEsKPv8KAQs7pcqoXweHLfrrT3/m03dvdWyS+wh6zJ2yDpewV22Vv2QEbMMl+R6vRw+hR/CGmuImnc2kcLTwP2H8V//oDjOnckw==</latexit> yt+1 yt = ⌘h rL(wt

    ), rSi ' ⌘h rL(w0 ), rSi progressive sharpening ] (Taylor ) (Taylor ) >0 sharpness ⌘hu, rL(wt )i <latexit sha1_base64="JG+q+egiDMSJxz/yvGmYgMdt5sA=">AAADJHiclVLLbhMxFPVMeZThlZYlG4uIqqVNNFNVhQ2oUlmwAFEEaSvVYXTHuUmt2p7B9kCiUX6Hr2GHkGDDt+CkEylJERJXsnR87rlPOyuksC6OfwXhyrXrN26u3opu37l7735jbf3Y5qXh2OG5zM1pBhal0Nhxwkk8LQyCyiSeZBeHE//JZzRW5PqDGxXYVTDQoi84OE+ljZ/DtHLbyZi26DB10cZz2mLogDIJeiCRMgXuPOvTcocyDZkE+npzxn1J3RZlphayaINZofDTf2WIt+h2zX/cXUw9f2nNR8yXnPX7fiHUj5I2mnE7nhq9CpIaNEltR+lasMd6OS8VasclWHuWxIXrVmCc4BLHESstFsAvYIBnHmpQaLvV9AnG9LFnerSfG3+0o1N2PqICZe1IZV456dMu+ybkX32ZWqrs+s+6ldBF6VDzy8L9UlKX08n70p4wyJ0ceQDcCN875edggDv/C6KIvUQ/nME3vtDbAg243DypGJiBEnrshx2wnQn6lxCGM6FHfs/J8lavguPddrLf3n+31zw4rDe+Sh6SR2STJOQpOSCvyBHpEB68CHqBCnT4NfwWfg9/XErDoI55QBYs/P0HOkX7iQ==</latexit> xt+1 xt = ' u <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> 2 r L(w0 ) 2 ⌘hu, rL(w0 ) + r L(wt )(wt = w0 )i ⌘S(wt )xt && S(w0 ) = 2/⌘ <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> t=0 stable hrL(w0 ), ui = 0 <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> Instability Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> 方 3 2 0 2 3 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS
  86. | Self-stabilization [Damian+ yt = ⌘h rL(wt ), rSi '

    ⌘h rL(w0 ), rSi <latexit sha1_base64="Iqu+raqFrpnJXvLeoggt2V8Gudo=">AAACzHicfVFNaxRBEO0Zv+IazUbRi5fGRYiaLDMSohchkBw8+BHRTQLpZajprZ006e4Zu2vUZVjw5H/MH/B32LO7gm5CChoer96jql7nlVaekuQ8iq9dv3Hz1srtzp3Vu/fWuuv3D31ZO4kDWerSHefgUSuLA1Kk8bhyCCbXeJSf7bX9o2/ovCrtF5pUODRQWDVWEihQWffnJGvoRTrlW3ySEX/DBRIIDbbQyLeEhVwDf7chDNBpPubfM3q2yRf0Z+HmOuGVwa8zK7/Km1zizbq9pJ/Mil8E6QL02KIOsvVoW4xKWRu0JDV4f5ImFQ0bcKSkxmlH1B4rkGdQ4EmAFgz6YTOLasqfBmbEx6ULzxKfsf86GjDeT0welO3afrnXkpf2crM0mcavh42yVU1o5XzwuNacSt7+Ax8ph5L0JACQToXduTwFB5LCb3U6Yh/DcQ7fh0EfK3RApXveCHCFUXYaji3EZouuEsKPv8KAQs7pcqoXweHLfrrT3/m03dvdWyS+wh6zJ2yDpewV22Vv2QEbMMl+R6vRw+hR/CGmuImnc2kcLTwP2H8V//oDjOnckw==</latexit> yt+1 progressive sharpening (Taylor ] ) >0 sharpness <latexit sha1_base64="ZYH6ckLoPYVPUyfk9OH5gT3olPI=">AAACdnicfVDbShxBEO2d3HRzW+NjIDRZJMboMhNEfRT0IS8Sg1kVnGWo6a1ZG7t7Jt01usuw/+DX5FV/wz/x0Z51A8kqKSg4nHOKqjppoaSjMLxpBE+ePnv+Ym6++fLV6zdvWwvvDl1eWoFdkavcHqfgUEmDXZKk8LiwCDpVeJSe7dT60TlaJ3Pzk0YF9jQMjMykAPJU0loZJhV9icZrw4R47KTGX3wtRgJ+sBxroNM04xcJffZy0mqHnXBS/CGIpqDNprWfLDTW434uSo2GhALnTqKwoF4FlqRQOG7GpcMCxBkM8MRDAxpdr5o8NeZLnunzLLe+DfEJ+/dEBdq5kU69s77TzWo1+aiW6pnNlG31KmmKktCI+8VZqTjlvE6M96VFQWrkAQgr/e1cnIIFQT7XZjPeRf+cxT2/6HuBFii3K1UMdqClGftnB/Fqjf5nhOEfo0c+52g21Yfg8Gsn2uhs/Fhvb+9ME59j79lHtswitsm22Te2z7pMsEv2m12x68Zt8CFYCj7dW4PGdGaR/VNBeAeDJsCk</latexit> xt+1 xt ' ⌘S(wt )xt ' ⌘[S(w0 ) + hrS, wt t+1 t <latexit sha1_base64="+xsPEdui7E+IPKPTsHSvJCKY/sw=">AAACt3icfVFdb9MwFHXC1wgf6+CRF4uKqWNrlaBpwAPSpPHAC2JodJtUR9GNe5NadZzMdmBV1B/JI7+EV5ysiNEhrmTp6NxzfXyP00oKY8Pwh+ffun3n7r2N+8GDh48eb/a2npyastYcx7yUpT5PwaAUCsdWWInnlUYoUoln6fyo7Z99RW1Eqb7YRYVxAbkSmeBgHZX05tvMiAIv6JChBTo5GbAC7CzN6Lck3KG7lElQuUTKFKQS6Mke/SOww2tipjthfJlYyliw/Y4OB9Fud+sisTuOTnr9cBR2RW+CaAX6ZFXHyZa3z6YlrwtUlkswZhKFlY0b0FZwicuA1QYr4HPIceKgggJN3HSpLOkLx0xpVmp3lKUde32igcKYRZE6ZbuFWe+15D97abHmbLM3cSNUVVtU/Mo4qyW1JW0jp1OhkVu5cAC4Fu7tlM9AA7fuY4KAvUe3nMaPzuhThRpsqV82DHReCLV0y+Zsr0X/E8Llb6FDLudoPdWb4PTVKDoYHXze7x8erRLfIM/IczIgEXlNDskHckzGhJPv5KdHPM9/6yd+5s+upL63mnlK/ir/4hf3hNTB</latexit> = (1 + ⌘y )x t x ' (1 + ⌘y )xt w0 i]xt (Taylor ) t ; 2 <latexit sha1_base64="SfEqv5FaCw2PA3UwjePBBT0BA0A=">AAACa3icfZDfShtBFMYn22rT+C+pd23BwSCosWFXxPZSsBfeSBUaFdywnJ2cxMGZ2XXmbElYctmn6W37MH2IvkMnMQWN0gMDP77zHc6cL82VdBSGvyvBi5cLi6+qr2tLyyura/XGmwuXFVZgR2Qqs1cpOFTSYIckKbzKLYJOFV6mt8eT/uU3tE5m5iuNcuxqGBjZlwLIS0l9Y5iU1IrGPHZS4x3/sB21YiTgo4R2hgkl9WbYDqfFn0I0gyab1VnSqBzEvUwUGg0JBc5dR2FO3RIsSaFwXIsLhzmIWxjgtUcDGl23nF4y5lte6fF+Zv0zxKfqw4kStHMjnXqnBrpx872J+Gwv1XObqf+pW0qTF4RG3C/uF4pTxicx8Z60KEiNPICw0v+dixuwIMiHWavFn9EfZ/HUL/qSowXK7G4Zgx1oacb+2EG8N6H/GWH4z+jJ5xzNp/oULvbb0WH78PygeXQ8S7zK3rFNts0i9pEdsRN2xjpMsO/sB/vJflX+BOvB2+D9vTWozGbW2aMKtv4CCPK72w==</latexit> > u <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> 2 r L(w0 ) && S(w0 ) = 2/⌘ <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> t=0 stable hrL(w0 ), ui = 0 <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> Instability Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> 飛 方 方 3 2 0 2 4 1 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS
  87. | Self-stabilization [Damian+ <latexit sha1_base64="DCwC0AB6tQAKX6cC7jSrGPTAtSM=">AAACvXicfZHfb9MwEMedANsoP9bBIy8WFdKGuiop0+CNSeOBBxBD0G3S3EUX95Jas53MdtiqqH8n4o9BwmnDrw5xkqWP7r723X2dllJYF0XfgvDW7Ttr6xt3O/fuP3i42d16dGyLynAc8UIW5jQFi1JoHDnhJJ6WBkGlEk/Si8OmfvIFjRWF/uxmJY4V5FpkgoPzqaR7yaxQeEkZOmASM8ck6FwiZRpSCfRTn+62+G6bKXDTNKNXSbSzyzIDvL5O3PlwPlxKzl/81lT9X7TDjMinjpnFy0m3Fw2iRdCbELfQI20cJVvBHpsUvFKoHZdg7VkclW5cg3GCS5x3WGWxBH4BOZ551KDQjuuFN3P6zGcmNCuMP9rRRfbPGzUoa2cq9cpmXrtaa5L/rKVqpbPLXo1rocvKoebLxlklqStoYzydCIPcyZkH4Eb42SmfgvfQ+e/pdNgb9MsZfO8bfSjRgCvM85qByZXQc79szvoN/U8I1z+FnrzP8aqrN+F4OIj3B/sf93oHh63jG+QJeUq2SUxekgPylhyREeHkK/kerAXr4esQQxnqpTQM2juPyV8RXv0Az1XZ/A==</latexit> <latexit sha1_base64="XhoBLzi1wOLbNv+TPf7EHWCAEJI=">AAACj3icfZBda9RAFIZnUz/q+rVbL70ZXISq7ZJIab1RFtYLBcWKblvoLOFk9iQdOpmEmRM1hP1B/hpv7b9xso2gW/HAwMN73uGc8yalVo7C8KIXbFy7fuPm5q3+7Tt3790fDLeOXFFZiTNZ6MKeJOBQK4MzUqTxpLQIeaLxODmftv3jL2idKsxnqkuc55AZlSoJ5KV4MK3jhp5FS77L65j4Sy6QQGgwmUa+KwwkGvi7bZEDnSUp/xrTkx3eyZ+EXfniwSgch6viVyHqYMS6OoyHvT2xKGSVoyGpwbnTKCxp3oAlJTUu+6JyWII8hwxPPRrI0c2b1bVL/tgrC54W1j9DfKX++aOB3Lk6T7yz3dqt91rxn70kX5tM6Yt5o0xZERp5OTitNKeCt1HyhbIoSdceQFrld+fyDCxI8oH3++I1+uMsvveDPpRogQr7tBFgs1yZpT82Ezst/c8I334bPfmco/VUr8LR83G0P97/uDeaTLvEN9lD9ohts4gdsAl7ww7ZjEn2nf1gP9lFMAwOglfB5NIa9Lo/D9hfFbz9Bc0JyIg=</latexit> yt+1 2 rL(wt

    ) ' rL(w0 ) + r L(w0 )(wt 2 <latexit sha1_base64="8uSa4n0wdODiLbonR7l3nrwKuCk=">AAACaXicfZDbSiNBEIY7sx6zHpL1RpSFZoMQFwkzIq6Xgl54oRhZE4VMDDWdStKku2fo7lkMQ658Gm/1aXyGfYntHASNiwUNH3/9RXX9USK4sb7/kvO+zM0vLC4t57+urK6tF4rf6iZONcMai0WsbyMwKLjCmuVW4G2iEWQk8Cbqn4z6N39QGx6raztIsCmhq3iHM7BOahW+hwoiAfQ3DbmiYR91eaLc7dPz3Vah5Ff8cdGPEEyhRKZVbRVzB2E7ZqlEZZkAYxqBn9hmBtpyJnCYD1ODCbA+dLHhUIFE08zGdwzpjlPatBNr95SlY/XtRAbSmIGMnFOC7ZnZ3kj8by+SM5tt56iZcZWkFhWbLO6kgtqYjkKiba6RWTFwAExz93fKeqCBWRdlPh+eojtO44VbdJmgBhvrn1kIuiu5Grpju+HeiD4zwv2r0ZHLOZhN9SPU9yvBYeXw6qB0fDJNfIlskx+kTALyixyTM1IlNcLIA3kkT+Q599crepve1sTq5aYzG+RdeaV/y0W6yw==</latexit> rS 2 ker(r L) u <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> && S(w0 ) = 2/⌘ 2 ⌧ yt = ⌘h rL(wt ), rSi ' ⌘ rS, rL(w0 ) <latexit sha1_base64="jF3VEmxBcTwjVE7nk8udiwvpT0o=">AAADAXicjVFNbxMxEPUuXyV8NIVjLxYRUlNKtBuqwrGoHBAfokikrdQN0awzm1q1vYs9C0SrnPg13BASJ34J/wZvukhtWhAjWXp6b55n/JwWSjqKol9BeOnylavXlq63bty8dXu5vXJnz+WlFTgQucrtQQoOlTQ4IEkKDwqLoFOF++nxTq3vf0TrZG7e0bTAoYaJkZkUQJ4atX8kBlIF/NVaooGO0ox/GlGXJ05q/MAvEKMuf9Dw7/tnhdNXPFx0ZBZE3G+Mj/7LuL7xF747aneiXjQvfh7EDeiwpnZHK8FmMs5FqdGQUODcYRwVNKzAkhQKZ62kdFiAOIYJHnpoQKMbVvN8Z/y+Z8Y8y60/hvicPe2oQDs31anvrJd0i1pNXqilemEyZU+GlTRFSWjEyeCsVJxyXn8eH0uLgtTUAxBW+t25OAKfLPkvbrWSZ+gfZ/G1H/SmQAuU2/UqATvR0sz8YyfJRo3+1Qif/zR65HOOF1M9D/b6vXirt/V2s7O90yS+xFbZPbbGYvaYbbPnbJcNmAhWg6fBi+Bl+CX8Gn4Lv5+0hkHjucvOVPjzN4pm70U=</latexit> r L(w0 ) t=0 stable 2 hrL(w0 ), ui = 0 立 3 2 0 2 高 3 小 5 3 r L(u, u) 1 3 w0 ) + r L(w0 )(wt w⇤ , wt 2 w t ' w0 Stage <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> 2 xt w⇤ ) <latexit sha1_base64="PCkDcUC+Zyxa1FPU8AMhgSzdCnY=">AAACanicfZDfShtBFMYnW60a+yfqlQgymAqllLBbxPZS0AtvShWMCm4IZydn4+DM7DpzthqW3Pk03urL+A4+hLMxoo3FAwM/vvMN55wvyZV0FIZ3teDd1PT7mdm5+vyHj58+NxYWD11WWIFtkanMHifgUEmDbZKk8Di3CDpReJScbVf9o79onczMAQ1y7GjoG5lKAeSlbmM11kCnScovusRjJzWe82cp7DaaYSscFX8N0RiabFx73YXaRtzLRKHRkFDg3EkU5tQpwZIUCof1uHCYgziDPp54NKDRdcrRIUO+7pUeTzPrnyE+Ul/+KEE7N9CJd1Y7usleJf63l+iJyZT+6pTS5AWhEY+D00JxyniVEu9Ji4LUwAMIK/3uXJyCBUE+y3o93kF/nMXfftCfHC1QZr+VMdi+lmboj+3H3yt6ywiXT0ZPPudoMtXXcPijFW22Nvc3mlvb48Rn2QpbY19ZxH6yLbbL9libCXbFrtkNu63dB4vBcrDyaA1q4z9L7J8KvjwAqXq8Nw==</latexit> Instability Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS ]
  88. | Self-stabilization [Damian+ <latexit sha1_base64="DCwC0AB6tQAKX6cC7jSrGPTAtSM=">AAACvXicfZHfb9MwEMedANsoP9bBIy8WFdKGuiop0+CNSeOBBxBD0G3S3EUX95Jas53MdtiqqH8n4o9BwmnDrw5xkqWP7r723X2dllJYF0XfgvDW7Ttr6xt3O/fuP3i42d16dGyLynAc8UIW5jQFi1JoHDnhJJ6WBkGlEk/Si8OmfvIFjRWF/uxmJY4V5FpkgoPzqaR7yaxQeEkZOmASM8ck6FwiZRpSCfRTn+62+G6bKXDTNKNXSbSzyzIDvL5O3PlwPlxKzl/81lT9X7TDjMinjpnFy0m3Fw2iRdCbELfQI20cJVvBHpsUvFKoHZdg7VkclW5cg3GCS5x3WGWxBH4BOZ551KDQjuuFN3P6zGcmNCuMP9rRRfbPGzUoa2cq9cpmXrtaa5L/rKVqpbPLXo1rocvKoebLxlklqStoYzydCIPcyZkH4Eb42SmfgvfQ+e/pdNgb9MsZfO8bfSjRgCvM85qByZXQc79szvoN/U8I1z+FnrzP8aqrN+F4OIj3B/sf93oHh63jG+QJeUq2SUxekgPylhyREeHkK/kerAXr4esQQxnqpTQM2juPyV8RXv0Az1XZ/A==</latexit> yt+1 ✓ <latexit sha1_base64="qSFFU1sZf3G4CopwKjmefxwqr14=">AAACi3icfZBda9RAFIZn41ddrW7rpTeDi1A/uiRraUUEC1XwRqzgtoXOGk5mT7JDZ5I4cyINIT/HX+OtXvhvnGxX0K14YODhPe/hzHmTUitHYfizF1y5eu36jbWb/Vu31+/cHWxsHrmishInstCFPUnAoVY5TkiRxpPSIphE43FydtD1j7+gdarIP1Jd4tRAlqtUSSAvxYNXddzQk6jdrmPiwimDn7lAAqExpS0BupwD3+Yi6bTUgmzOY/o0bsfCqmxOj+LBMByFi+KXIVrCkC3rMN7o7YhZISuDOUkNzp1GYUnTBiwpqbHti8phCfIMMjz1mINBN20Wl7b8oVdmPC2sfznxhfrnRAPGudok3mmA5m6114n/7CVmZTOlz6eNysuKMJcXi9NKcyp4FyOfKYuSdO0BpFX+71zOwedDPux+X7xGf5zFd37R+xItUGEfNwJsZlTe+mMz8bSj/xnh/LfRk885Wk31MhyNR9HuaPfDznD/YJn4GrvPHrAtFrE9ts/eskM2YZJ9Zd/Yd/YjWA+eBS+ClxfWoLecucf+quDNL4q7yJs=</latexit> ◆

    2 xt 2 yt ' ⌘ ↵ yt+1 progressive sharpening u 2 <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> r L(w0 ) ⌧ yt = ⌘h rL(wt ), rSi ' ⌘ rS, rL(w0 ) <latexit sha1_base64="XhoBLzi1wOLbNv+TPf7EHWCAEJI=">AAACj3icfZBda9RAFIZnUz/q+rVbL70ZXISq7ZJIab1RFtYLBcWKblvoLOFk9iQdOpmEmRM1hP1B/hpv7b9xso2gW/HAwMN73uGc8yalVo7C8KIXbFy7fuPm5q3+7Tt3790fDLeOXFFZiTNZ6MKeJOBQK4MzUqTxpLQIeaLxODmftv3jL2idKsxnqkuc55AZlSoJ5KV4MK3jhp5FS77L65j4Sy6QQGgwmUa+KwwkGvi7bZEDnSUp/xrTkx3eyZ+EXfniwSgch6viVyHqYMS6OoyHvT2xKGSVoyGpwbnTKCxp3oAlJTUu+6JyWII8hwxPPRrI0c2b1bVL/tgrC54W1j9DfKX++aOB3Lk6T7yz3dqt91rxn70kX5tM6Yt5o0xZERp5OTitNKeCt1HyhbIoSdceQFrld+fyDCxI8oH3++I1+uMsvveDPpRogQr7tBFgs1yZpT82Ezst/c8I334bPfmco/VUr8LR83G0P97/uDeaTLvEN9lD9ohts4gdsAl7ww7ZjEn2nf1gP9lFMAwOglfB5NIa9Lo/D9hfFbz9Bc0JyIg=</latexit> 2 力 3 2 0 2 6 3 r L(u, u) ↵ := hrS, rLi 2 := krSk <latexit sha1_base64="M/RIeDTgA9+KlWxjKPyg62RBgcU=">AAACd3icfZDfShtBFMYnq/VP2tpoL71waFCk2rArolIQBHvhhVKLRgUnhLOTk83gzOwyMyuGJQ/h03hrH6OP0jsncQWN4oGBH993DmfOF2dSWBeG/yrBxOSHqemZ2erHT5/nvtTmF85smhuOTZ7K1FzEYFEKjU0nnMSLzCCoWOJ5fLU/9M+v0ViR6lPXz7ClINGiKzg4L7Vrawxk1gP6c5cyCTqRSJmGWAI9Wf9R0iEzI6ddq4eNcFT0NUQl1ElZx+35yibrpDxXqB2XYO1lFGauVYBxgkscVFluMQN+BQleetSg0LaK0VUDuuyVDu2mxj/t6Eh9PlGAsravYt+pwPXsuDcU3/RiNbbZdXdahdBZ7lDzx8XdXFKX0mFktCMMcif7HoAb4f9OeQ8McOeDrVbZL/THGTzyi35naMCl5nvBwCRK6IE/NmHrQ3qvEW6eGj35nKPxVF/D2UYj2mps/dms7+2Xic+QRfKNrJKIbJM9ckCOSZNwckvuyD35W/kfLAUrwepja1ApZ76SFxVED4XmwJs=</latexit> <latexit sha1_base64="w34K6vB5MtbA/uqgO9CL2ZFmMfs=">AAACY3icfZDRSltBEIY3p9pqtDXaghdSWBoKpZRwTgi2FAqCXngjRtpEwU3DnM0kLu7uOezOKQ3HPE1v6wP5AH2PbmIEjaUDCx///MPs/Gmulac4vqlET5aWnz5bWa2urT9/sVHb3Or6rHASOzLTmTtLwaNWFjukSONZ7hBMqvE0vdyf9k9/oPMqs99onGPPwMiqoZJAQerXtkWKBPzzFy6uhIVUA/8qrr43+7V63IhnxR9DMoc6m1e7v1lpiUEmC4OWpAbvz5M4p14JjpTUOKmKwmMO8hJGeB7QgkHfK2cXTPjboAz4MHPhWeIz9f5ECcb7sUmD0wBd+MXeVPxnLzULm2n4qVcqmxeEVt4uHhaaU8an8fCBcihJjwOAdCr8ncsLcCAphFitigMMxzk8CouOc3RAmXtfCnAjo+wkHDsSH6b0PyP8vDMGCjkni6k+hm6zkew2dk9a9b39eeIrbIe9Ye9Ywj6yPXbI2qzDJJuwX+w3u678idairejVrTWqzGdesgcVvf4LThq4xg==</latexit> sharpness && S(w0 ) = 2/⌘ <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> t=0 2 xt stable hrL(w0 ), ui = 0 <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> Instability Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS ]
  89. | Self-stabilization [Damian+ <latexit sha1_base64="DCwC0AB6tQAKX6cC7jSrGPTAtSM=">AAACvXicfZHfb9MwEMedANsoP9bBIy8WFdKGuiop0+CNSeOBBxBD0G3S3EUX95Jas53MdtiqqH8n4o9BwmnDrw5xkqWP7r723X2dllJYF0XfgvDW7Ttr6xt3O/fuP3i42d16dGyLynAc8UIW5jQFi1JoHDnhJJ6WBkGlEk/Si8OmfvIFjRWF/uxmJY4V5FpkgoPzqaR7yaxQeEkZOmASM8ck6FwiZRpSCfRTn+62+G6bKXDTNKNXSbSzyzIDvL5O3PlwPlxKzl/81lT9X7TDjMinjpnFy0m3Fw2iRdCbELfQI20cJVvBHpsUvFKoHZdg7VkclW5cg3GCS5x3WGWxBH4BOZ551KDQjuuFN3P6zGcmNCuMP9rRRfbPGzUoa2cq9cpmXrtaa5L/rKVqpbPLXo1rocvKoebLxlklqStoYzydCIPcyZkH4Eb42SmfgvfQ+e/pdNgb9MsZfO8bfSjRgCvM85qByZXQc79szvoN/U8I1z+FnrzP8aqrN+F4OIj3B/sf93oHh63jG+QJeUq2SUxekgPylhyREeHkK/kerAXr4esQQxnqpTQM2juPyV8RXv0Az1XZ/A==</latexit> ⌧ yt = ⌘h rL(wt

    ), rSi ' ⌘ rS, rL(w0 ) <latexit sha1_base64="XhoBLzi1wOLbNv+TPf7EHWCAEJI=">AAACj3icfZBda9RAFIZnUz/q+rVbL70ZXISq7ZJIab1RFtYLBcWKblvoLOFk9iQdOpmEmRM1hP1B/hpv7b9xso2gW/HAwMN73uGc8yalVo7C8KIXbFy7fuPm5q3+7Tt3790fDLeOXFFZiTNZ6MKeJOBQK4MzUqTxpLQIeaLxODmftv3jL2idKsxnqkuc55AZlSoJ5KV4MK3jhp5FS77L65j4Sy6QQGgwmUa+KwwkGvi7bZEDnSUp/xrTkx3eyZ+EXfniwSgch6viVyHqYMS6OoyHvT2xKGSVoyGpwbnTKCxp3oAlJTUu+6JyWII8hwxPPRrI0c2b1bVL/tgrC54W1j9DfKX++aOB3Lk6T7yz3dqt91rxn70kX5tM6Yt5o0xZERp5OTitNKeCt1HyhbIoSdceQFrld+fyDCxI8oH3++I1+uMsvveDPpRogQr7tBFgs1yZpT82Ezst/c8I334bPfmco/VUr8LR83G0P97/uDeaTLvEN9lD9ohts4gdsAl7ww7ZjEn2nf1gP9lFMAwOglfB5NIa9Lo/D9hfFbz9Bc0JyIg=</latexit> yt+1 ✓ <latexit sha1_base64="qSFFU1sZf3G4CopwKjmefxwqr14=">AAACi3icfZBda9RAFIZn41ddrW7rpTeDi1A/uiRraUUEC1XwRqzgtoXOGk5mT7JDZ5I4cyINIT/HX+OtXvhvnGxX0K14YODhPe/hzHmTUitHYfizF1y5eu36jbWb/Vu31+/cHWxsHrmishInstCFPUnAoVY5TkiRxpPSIphE43FydtD1j7+gdarIP1Jd4tRAlqtUSSAvxYNXddzQk6jdrmPiwimDn7lAAqExpS0BupwD3+Yi6bTUgmzOY/o0bsfCqmxOj+LBMByFi+KXIVrCkC3rMN7o7YhZISuDOUkNzp1GYUnTBiwpqbHti8phCfIMMjz1mINBN20Wl7b8oVdmPC2sfznxhfrnRAPGudok3mmA5m6114n/7CVmZTOlz6eNysuKMJcXi9NKcyp4FyOfKYuSdO0BpFX+71zOwedDPux+X7xGf5zFd37R+xItUGEfNwJsZlTe+mMz8bSj/xnh/LfRk885Wk31MhyNR9HuaPfDznD/YJn4GrvPHrAtFrE9ts/eskM2YZJ9Zd/Yd/YjWA+eBS+ClxfWoLecucf+quDNL4q7yJs=</latexit> ◆ 2 xt 2 yt ' ⌘ ↵ yt+1 xt+1 ' (1 + ⌘yt )xt Stage yt < 0 2 xt 2 3 r L(u, u) ↵ := hrS, rLi 2 := krSk <latexit sha1_base64="M/RIeDTgA9+KlWxjKPyg62RBgcU=">AAACd3icfZDfShtBFMYnq/VP2tpoL71waFCk2rArolIQBHvhhVKLRgUnhLOTk83gzOwyMyuGJQ/h03hrH6OP0jsncQWN4oGBH993DmfOF2dSWBeG/yrBxOSHqemZ2erHT5/nvtTmF85smhuOTZ7K1FzEYFEKjU0nnMSLzCCoWOJ5fLU/9M+v0ViR6lPXz7ClINGiKzg4L7Vrawxk1gP6c5cyCTqRSJmGWAI9Wf9R0iEzI6ddq4eNcFT0NUQl1ElZx+35yibrpDxXqB2XYO1lFGauVYBxgkscVFluMQN+BQleetSg0LaK0VUDuuyVDu2mxj/t6Eh9PlGAsravYt+pwPXsuDcU3/RiNbbZdXdahdBZ7lDzx8XdXFKX0mFktCMMcif7HoAb4f9OeQ8McOeDrVbZL/THGTzyi35naMCl5nvBwCRK6IE/NmHrQ3qvEW6eGj35nKPxVF/D2UYj2mps/dms7+2Xic+QRfKNrJKIbJM9ckCOSZNwckvuyD35W/kfLAUrwepja1ApZ76SFxVED4XmwJs=</latexit> <latexit sha1_base64="w34K6vB5MtbA/uqgO9CL2ZFmMfs=">AAACY3icfZDRSltBEIY3p9pqtDXaghdSWBoKpZRwTgi2FAqCXngjRtpEwU3DnM0kLu7uOezOKQ3HPE1v6wP5AH2PbmIEjaUDCx///MPs/Gmulac4vqlET5aWnz5bWa2urT9/sVHb3Or6rHASOzLTmTtLwaNWFjukSONZ7hBMqvE0vdyf9k9/oPMqs99onGPPwMiqoZJAQerXtkWKBPzzFy6uhIVUA/8qrr43+7V63IhnxR9DMoc6m1e7v1lpiUEmC4OWpAbvz5M4p14JjpTUOKmKwmMO8hJGeB7QgkHfK2cXTPjboAz4MHPhWeIz9f5ECcb7sUmD0wBd+MXeVPxnLzULm2n4qVcqmxeEVt4uHhaaU8an8fCBcihJjwOAdCr8ncsLcCAphFitigMMxzk8CouOc3RAmXtfCnAjo+wkHDsSH6b0PyP8vDMGCjkni6k+hm6zkew2dk9a9b39eeIrbIe9Ye9Ywj6yPXbI2qzDJJuwX+w3u678idairejVrTWqzGdesgcVvf4LThq4xg==</latexit> <latexit sha1_base64="SfEqv5FaCw2PA3UwjePBBT0BA0A=">AAACa3icfZDfShtBFMYn22rT+C+pd23BwSCosWFXxPZSsBfeSBUaFdywnJ2cxMGZ2XXmbElYctmn6W37MH2IvkMnMQWN0gMDP77zHc6cL82VdBSGvyvBi5cLi6+qr2tLyyura/XGmwuXFVZgR2Qqs1cpOFTSYIckKbzKLYJOFV6mt8eT/uU3tE5m5iuNcuxqGBjZlwLIS0l9Y5iU1IrGPHZS4x3/sB21YiTgo4R2hgkl9WbYDqfFn0I0gyab1VnSqBzEvUwUGg0JBc5dR2FO3RIsSaFwXIsLhzmIWxjgtUcDGl23nF4y5lte6fF+Zv0zxKfqw4kStHMjnXqnBrpx872J+Gwv1XObqf+pW0qTF4RG3C/uF4pTxicx8Z60KEiNPICw0v+dixuwIMiHWavFn9EfZ/HUL/qSowXK7G4Zgx1oacb+2EG8N6H/GWH4z+jJ5xzNp/oULvbb0WH78PygeXQ8S7zK3rFNts0i9pEdsRN2xjpMsO/sB/vJflX+BOvB2+D9vTWozGbW2aMKtv4CCPK72w==</latexit> Stage |xt | <latexit sha1_base64="g/mTn1K11DNKPTFXx/AdimVzT24=">AAACT3icfZBNSwMxEIaz9bt+69HLYhFEpOyKqMeCHryIClYFt5TZdFqDSXZJZqVl7W/wqj/Lo7/Em5jWFbSKA4GHd94wM2+cSmEpCF690tj4xOTU9Ex5dm5+YXFpeeXSJpnhWOeJTMx1DBal0FgnQRKvU4OgYolX8d3hoH91j8aKRF9QL8WGgo4WbcGBnFR/6DbpoblUCarBsPzfEBZQYUWdNZe93aiV8EyhJi7B2pswSKmRgyHBJfbLUWYxBX4HHbxxqEGhbeTDbfv+hlNafjsx7mnyh+r3Hzkoa3sqdk4FdGtHewPxz16sRiZT+6CRC51mhJp/Dm5n0qfEH0Tht4RBTrLnALgRbnef34IBTi6wcjk6QnecwRM36DRFA5SYrTwC01FC992xnWh7QP8ZoftldORyDkdT/Q2XO9Vwr7p3vlupHRaJT7M1ts42Wcj2WY0dszNWZ5wJ9sie2LP34r1576XCWvIKWGU/qjTzAbXTtGY=</latexit> yt > 0 <latexit sha1_base64="PZp6Ze5paBIbCGn9YhZiCRm/v1A=">AAACT3icfZBNSwMxEIaz9bt+Vo9eFosgImVXRD1JQQ9eRAWrglvKbDqtwSS7JLNiWfobvOrP8ugv8SamdQWt4kDg4Z03zMwbp1JYCoJXrzQ2PjE5NT1Tnp2bX1hcqixf2iQzHBs8kYm5jsGiFBobJEjidWoQVCzxKr47HPSv7tFYkegL6qXYVNDVoiM4kJMavRYdBK2lalALhuX/hrCAKivqrFXxdqJ2wjOFmrgEa2/CIKVmDoYEl9gvR5nFFPgddPHGoQaFtpkPt+37605p+53EuKfJH6rff+SgrO2p2DkV0K0d7Q3EP3uxGplMnf1mLnSaEWr+ObiTSZ8SfxCF3xYGOcmeA+BGuN19fgsGOLnAyuXoCN1xBk/coNMUDVBiNvMITFcJ3XfHdqOtAf1nhIcvoyOXczia6m+43K6Fu7Xd851q/bBIfJqtsjW2wUK2x+rsmJ2xBuNMsEf2xJ69F+/Ney8V1pJXwAr7UaWZD7DCs90=</latexit> <latexit sha1_base64="ZU6NU9Ok6hQVvITT60poTUqwEJU=">AAACUXicfZDPShxBEMZrJjGa1ST+OebSuAREwjITRHPIQdCDF1HBdQVnWWp6a9dmu3uG7pqQZdiH8BofK6c8ijd71xWSNaSg4cdXX1NVX15q5TlJfkfxq9dLb5ZX3jZW1969/7C+sXnli8pJastCF+46R09aWWqzYk3XpSM0uaZOPjqa9jvfyXlV2Esel9Q1OLRqoCRykDrjHotvIumtN5NWMivxEtI5NGFe572NaC/rF7IyZFlq9P4mTUru1uhYSU2TRlZ5KlGOcEg3AS0a8t16tu9EfApKXwwKF55lMVP//FGj8X5s8uA0yLd+sTcV/9nLzcJkHnzt1sqWFZOVT4MHlRZciGkYoq8cSdbjACidCrsLeYsOJYfIGo3smMJxjk7DoLOSHHLhdusM3dAoOwnHDrPPU/qfEX88GwOFnNPFVF/C1ZdWut/av9hrHh7NE1+Bj7ANO5DCARzCCZxDGySM4A5+wn30K3qIIY6frHE0/7MFf1W8+ghR2bMw</latexit> |xt | <latexit sha1_base64="g/mTn1K11DNKPTFXx/AdimVzT24=">AAACT3icfZBNSwMxEIaz9bt+69HLYhFEpOyKqMeCHryIClYFt5TZdFqDSXZJZqVl7W/wqj/Lo7/Em5jWFbSKA4GHd94wM2+cSmEpCF690tj4xOTU9Ex5dm5+YXFpeeXSJpnhWOeJTMx1DBal0FgnQRKvU4OgYolX8d3hoH91j8aKRF9QL8WGgo4WbcGBnFR/6DbpoblUCarBsPzfEBZQYUWdNZe93aiV8EyhJi7B2pswSKmRgyHBJfbLUWYxBX4HHbxxqEGhbeTDbfv+hlNafjsx7mnyh+r3Hzkoa3sqdk4FdGtHewPxz16sRiZT+6CRC51mhJp/Dm5n0qfEH0Tht4RBTrLnALgRbnef34IBTi6wcjk6QnecwRM36DRFA5SYrTwC01FC992xnWh7QP8ZoftldORyDkdT/Q2XO9Vwr7p3vlupHRaJT7M1ts42Wcj2WY0dszNWZ5wJ9sie2LP34r1576XCWvIKWGU/qjTzAbXTtGY=</latexit> 2 u <latexit sha1_base64="xz1rrCAYZalE2QIhaHMFsO1wX5g=">AAACYnicfZDfShtBFMZn1/7Rba1JBW/ai6GhoKWEXRHbS0EvvLDUgtFAdhvOTs4mQ2Zml5lZNWzzMr3VF/LeB3E2plCjeGDgx3e+4ZzzpYXgxobhjecvvXj56vXySvDm7eq7tUbz/anJS82ww3KR624KBgVX2LHcCuwWGkGmAs/S8X7dPztHbXiuTuykwETCUPGMM7BO6jc2YgWpgN/bR5uxBDtKM3rRD7f6jVbYDmdFH0M0hxaZ13G/6e3Eg5yVEpVlAozpRWFhkwq05UzgNIhLgwWwMQyx51CBRJNUswOm9LNTBjTLtXvK0pn6/48KpDETmTpnvaRZ7NXik71ULky22fek4qooLSp2PzgrBbU5rdOhA66RWTFxAExztztlI9DArMswCOIDdMdp/OEG/SxQg831lyoGPZRcTd2xw/hrTc8Z4fKf0ZHLOVpM9TGcbrej3fbur53W3v488WXygXwimyQi38geOSTHpEMY+UP+kity7d36gd/01++tvjf/s04elP/xDqpduHs=</latexit> rS <latexit sha1_base64="W2HLbqXF8AlENmirO1Zf56ObKPE=">AAACUnicfVJNSwMxEM3W71q/j16CRRCRsiulehT04EVUtB/QLWU2nbahSXZJsmJZ+ie86s/y4l/xZForaBUHAo83b3gzj0SJ4Mb6/puXm5tfWFxaXsmvFtbWNza3tmsmTjXDKotFrBsRGBRcYdVyK7CRaAQZCaxHg/Nxv/6A2vBY3dthgi0JPcW7nIF1VCNUEAmgd+3Nol/yJ0V/g2AKimRaN+0trxx2YpZKVJYJMKYZ+IltZaAtZwJH+TA1mAAbQA+bDiqQaFrZZOER3XdMh3Zj7Z6ydMJ+n8hAGjOUkVNKsH0z2xuTf/YiOeNsu6etjKsktajYp3E3FdTGdJwG7XCNzIqhA8A0d7tT1gcNzLrM8vnwAt1xGq+c0XWCGmysD7MQdE9yNXLH9sKjMfpPCI9fQodczsFsqr9B7bgUVEqV23Lx7Hya+DLZJXvkgATkhJyRS3JDqoQRQZ7IM3nxXr33nPsln9KcN53ZIT8qV/gAD6+0Dw==</latexit> <latexit sha1_base64="CqobUnoWNXmeWYEfdLU80lEqUqc=">AAACU3icfZBNSwMxEIaza/2q33r0EiyCiJRdEfUo6MGLqGBrxS0ym87WYJJdkqxYlv4Kr/qzPPhbvJitFbQVBwIP77xhZt44E9zYIHj3/InK5NT0zGx1bn5hcWl5ZbVp0lwzbLBUpLoVg0HBFTYstwJbmUaQscDr+OG47F8/ojY8VVe2l2FbQlfxhDOwTrqJJNj7OKH53XItqAeDouMQDqFGhnVxt+LtRZ2U5RKVZQKMuQ2DzLYL0JYzgf1qlBvMgD1AF28dKpBo2sVg4z7ddEqHJql2T1k6UH/+KEAa05Oxc5YbmtFeKf7Zi+XIZJsctguustyiYl+Dk1xQm9IyDtrhGpkVPQfANHe7U3YPGph1oVWr0Qm64zSeuUHnGWqwqd4uItBdyVXfHduNdkr6zwhP30ZHLudwNNVxaO7Ww/36/uVe7eh4mPgMWScbZIuE5IAckVNyQRqEEUmeyQt59d68D9/3K19W3xv+WSO/yl/4BFXZtK8=</latexit> && S(w0 ) = 2/⌘ 2 <latexit sha1_base64="3e9cmUqAth2lgmfHr1qH8FvfY38=">AAACY3icfZDfShtBFMYnW9tq7J9ohV5IYWgoqEi6KyH2RhDsRW9Ei40KbghnJ2fj4MzsMnO2NSx5Gm/1gXwA36OzMYKNxQMDP77zDeecL8mVdBSGt7XgxdzLV6/nF+qLb96+e99YWj52WWEFdkWmMnuagEMlDXZJksLT3CLoROFJcrFX9U9+o3UyM79olGNPw9DIVAogL/UbH4/WYg10nqT8Tz9c5zt862uMBP1GM2yFk+JPIZpCk03rsL9Ua8eDTBQaDQkFzp1FYU69EixJoXBcjwuHOYgLGOKZRwMaXa+cXDDmX7wy4Glm/TPEJ+rjHyVo50Y68c5qWzfbq8T/9hI9M5nSb71SmrwgNOJ+cFooThmv4uEDaVGQGnkAYaXfnYtzsCDIh1ivx9/RH2dx3w86yNECZXajjMEOtTRjf+ww3qzoOSNcPhg9+Zyj2VSfwvFWK+q0Oj/bzd29aeLzbJV9ZmssYttsl/1gh6zLBBuzK3bNbmp3wWKwHKzcW4Pa9M8H9k8Fn/4C+ia4Fg==</latexit> r L(w0 ) t=0 stable hrL(w0 ), ui = 0 <latexit sha1_base64="m7yeepS2NZfZ0Qf29JmlNyLgvTc=">AAACfnicfVDRThNBFL1dRLGgFnz0ZWKDQYNllxDkxYQEH3jQgIkFEqZp7k7vLhNmZjczs2qz6X/4Nb7qL/g3zLYlkWI8ySRnzj03996Tlko6H8d/WtHSg+WHj1Yet1fXnjx91lnfOHNFZQX1RaEKe5GiIyUN9b30ii5KS6hTRefp9VFTP/9K1snCfPHjkgYacyMzKdAHadjZ5QpNrohxg6lC9nGLa/RXaca+DePX27efituZ7T2Lh51u3IunYPdJMiddmON0uN7a46NCVJqMFwqdu0zi0g9qtF4KRZM2rxyVKK4xp8tADWpyg3p63IRtBmXEssKGZzybqn931KidG+s0OJtl3WKtEf9ZS/XCZJ8dDGppysqTEbPBWaWYL1iTHBtJS8KrcSAorAy7M3GFFoUP+bbb/AOF4yx9CoNOSrLoC/um5mhzLc0kHJvz7Yb9z4jfb42BhZyTxVTvk7PdXrLf2/+81z08mie+Ai/gJWxBAu/gEI7hFPog4Af8hF/wO4LoVfQ22plZo9a85zncQXRwAzFxwiQ=</latexit> Instability Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> 大 3 2 0 2 2 3 7 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS ]
  90. xt+1 ' ( <latexit sha1_base64="SfEqv5FaCw2PA3UwjePBBT0BA0A=">AAACa3icfZDfShtBFMYn22rT+C+pd23BwSCosWFXxPZSsBfeSBUaFdywnJ2cxMGZ2XXmbElYctmn6W37MH2IvkMnMQWN0gMDP77zHc6cL82VdBSGvyvBi5cLi6+qr2tLyyura/XGmwuXFVZgR2Qqs1cpOFTSYIckKbzKLYJOFV6mt8eT/uU3tE5m5iuNcuxqGBjZlwLIS0l9Y5iU1IrGPHZS4x3/sB21YiTgo4R2hgkl9WbYDqfFn0I0gyab1VnSqBzEvUwUGg0JBc5dR2FO3RIsSaFwXIsLhzmIWxjgtUcDGl23nF4y5lte6fF+Zv0zxKfqw4kStHMjnXqnBrpx872J+Gwv1XObqf+pW0qTF4RG3C/uF4pTxicx8Z60KEiNPICw0v+dixuwIMiHWavFn9EfZ/HUL/qSowXK7G4Zgx1oacb+2EG8N6H/GWH4z+jJ5xzNp/oULvbb0WH78PygeXQ8S7zK3rFNts0i9pEdsRN2xjpMsO/sB/vJflX+BOvB2+D9vTWozGbW2aMKtv4CCPK72w==</latexit> (1 + ⌘yt )xt ✓

    <latexit sha1_base64="qSFFU1sZf3G4CopwKjmefxwqr14=">AAACi3icfZBda9RAFIZn41ddrW7rpTeDi1A/uiRraUUEC1XwRqzgtoXOGk5mT7JDZ5I4cyINIT/HX+OtXvhvnGxX0K14YODhPe/hzHmTUitHYfizF1y5eu36jbWb/Vu31+/cHWxsHrmishInstCFPUnAoVY5TkiRxpPSIphE43FydtD1j7+gdarIP1Jd4tRAlqtUSSAvxYNXddzQk6jdrmPiwimDn7lAAqExpS0BupwD3+Yi6bTUgmzOY/o0bsfCqmxOj+LBMByFi+KXIVrCkC3rMN7o7YhZISuDOUkNzp1GYUnTBiwpqbHti8phCfIMMjz1mINBN20Wl7b8oVdmPC2sfznxhfrnRAPGudok3mmA5m6114n/7CVmZTOlz6eNysuKMJcXi9NKcyp4FyOfKYuSdO0BpFX+71zOwedDPux+X7xGf5zFd37R+xItUGEfNwJsZlTe+mMz8bSj/xnh/LfRk885Wk31MhyNR9HuaPfDznD/YJn4GrvPHrAtFrE9ts/eskM2YZJ9Zd/Yd/YjWA+eBS+ClxfWoLecucf+quDNL4q7yJs=</latexit> yt ' ⌘ ↵ yt+1 [Damian+ <latexit sha1_base64="I08CI+X67LS8Tp0IIeEpq3Uc6XM=">AAACk3icfVBtaxNBEN6cb+350lTxk18WgyCi4S6UKohQ2oJ+KVYw7ZVuDHObucvSvb1jd08MR35Sf42fCvpfnCQnaCoOLDwz8zwzO09aaeV8FF11ghs3b92+s7EZ3r13/8FWd/vhiStrK3EoS13aJAWHWhkceuU1JpVFKFKNp+nFwaJ/+hWtU6X57GcVjgrIjcqUBE+lcfe9SDFXppE0w81DMSk9T/g7npxxIVbpGaUCdDWFVyKzIEnhYZB8GYQCzaRVjru9qB8tg18HcQt6rI3j8XZnh6bLukDjpQbnzuOo8qMGrFdSI32ldliBvIAczwkaKNCNmuXFc/6MKhOelZae8XxZ/VPRQOHcrEiJWYCfuvXeovjPXlqsbfbZm1GjTFV7NHK1OKs19yVf2MknyqL0ekYApFX0dy6nQC55Mj0MxSHScRaPaNHHCi340r5oBNi8UGZOx+bi5QL9jwjffhMJkc/xuqvXwcmgH+/2dz/t9PYOWsc32BP2lD1nMXvN9tgHdsyGTLJL9p39YD+Dx8HbYD84XFGDTqt5xP6K4OgXAsjKXg==</latexit> ◆ 2 xt 2 X   <latexit sha1_base64="epB8noL4M6eS8d8kUWCYUgUBfVw=">AAACnHicfVBdaxNBFJ2sXzV+NNVHQQaDUEXCrpS2CEKhfRCkGMGkkU5Y7k5utkNnZpeZu5Kwzd/yvwi+6u9wNo1iU/HCDIdzz+Xce7JSK09x/K0V3bh56/adjbvte/cfPNzsbD0a+qJyEgey0IUbZeBRK4sDUqRxVDoEk2k8yc4Pm/7JF3ReFfYTzUscG8itmioJFKi00xcZ5srWmQFyarYYbdMLIT43P9rJH5q/ecvXlBezlC6EmKd0RZl2unEvXha/DpIV6LJV9dOt1o6YFLIyaElq8P40iUsa1+BISY2Ltqg8liDPIcfTAC0Y9ON6efqCPw/MhE8LF54lvmT/nqjBeD83WVCGBc/8eq8h/9nLzJozTffHtbJlRWjlpfG00pwK3uTKJ8qhJD0PAKRTYXcuz8CBpJB+uy2OMBzn8DgYfSjRARXuZS3A5UbZRTg2F68a9D8hzH4LYZlzsp7qdTB83Ut2e7sfd7oHh6vEN9gT9oxts4TtsQP2jvXZgEn2lX1nP9jP6Gl0FL2Pji+lUWs185hdqWj4C1w/0do=</latexit> X(t) |xt | := Y (t) yt Ẋ = XY 2 Ẏ = ↵ 2 X ] yt <latexit sha1_base64="1//p2HkB9b8P3fSMRvIrUVAbbMU=">AAACTXicfZBNSwMxEIaz9bt+69HLYhFEpOyKqEdBD15ERauCW8psOl1Dk+ySzIpl6U/wqj/Lsz/Em4hpraBVHAg8vPOGmXnjTApLQfDilUZGx8YnJqfK0zOzc/MLi0uXNs0NxxpPZWquY7AohcYaCZJ4nRkEFUu8itsHvf7VHRorUn1BnQzrChItWoIDOem806DGQiWoBv3yf0M4gAob1Glj0duOminPFWriEqy9CYOM6gUYElxitxzlFjPgbUjwxqEGhbZe9Hft+mtOafqt1Linye+r338UoKztqNg5FdCtHe71xD97sRqaTK29eiF0lhNq/jm4lUufUr8XhN8UBjnJjgPgRrjdfX4LBji5uMrl6BDdcQaP3aCTDA1QajaKCEyihO66Y5Nos0f/GeH+y+jI5RwOp/obLreq4U5152y7sn8wSHySrbBVts5Ctsv22RE7ZTXGWcIe2CN78p69V+/Ne/+0lrzBn2X2o0oTH7LhtFo=</latexit> =2 xt <latexit sha1_base64="OLiKG9AWP6m0Od3isOVdG94OD4M=">AAACTXicfZBNSwMxEIaz9bt+69HLYhFEpOyKqEdBD15ERauCW8psOl1Dk+ySzIpl6U/wqj/Lsz/Em4hpraBVHAg8vPOGmXnjTApLQfDilUZGx8YnJqfK0zOzc/MLi0uXNs0NxxpPZWquY7AohcYaCZJ4nRkEFUu8itsHvf7VHRorUn1BnQzrChItWoIDOen8vkGNhUpQDfrl/4ZwABU2qNPGorcdNVOeK9TEJVh7EwYZ1QswJLjEbjnKLWbA25DgjUMNCm296O/a9dec0vRbqXFPk99Xv/8oQFnbUbFzKqBbO9zriX/2YjU0mVp79ULoLCfU/HNwK5c+pX4vCL8pDHKSHQfAjXC7+/wWDHBycZXL0SG64wweu0EnGRqg1GwUEZhECd11xybRZo/+M8L9l9GRyzkcTvU3XG5Vw53qztl2Zf9gkPgkW2GrbJ2FbJftsyN2ymqMs4Q9sEf25D17r96b9/5pLXmDP8vsR5UmPgCw+LRZ</latexit> Y = sharpening Y = Alex Damian X =2 Instability <latexit sha1_base64="Je3TUjQFY3Rxci9aQNkPwPfvzMU=">AAACbHicfZDfalNBEMY3p9XW+C9tvSvCYlBEJJwjpS1CoVAvvBErmLbQDWHOZpIs3X/dnVMMh9z6NN7qu/gSPoObNEJNxYGFH998w+x8pdcqUp7/bGQrq3furq3fa95/8PDR49bG5kl0VZDYlU67cFZCRK0sdkmRxjMfEEyp8bS8OJr1T68wROXsZ5p47BkYWTVUEihJ/RYXVxDQR6Wd5W8PuEACLuJlIC5A+zH0W+28k8+L34ZiAW22qOP+RmNHDJysDFqSGmI8L3JPvRoCKalx2hRVRA/yAkZ4ntCCwdir56dM+fOkDPjQhfQs8bl6c6IGE+PElMlpgMZxuTcT/9krzdJmGu73amV9RWjl9eJhpTk5PsuJD1RASXqSAGRQ6e9cjiGApJRmsyneYTou4Ie06KPHAOTCq1pAGBllp+nYkXg9o/8Z4csfY6KUc7Gc6m04edMpdju7n3bah0eLxNfZNnvGXrKC7bFD9p4dsy6T7Cv7xr6zH41f2ZNsO3t6bc0ai5kt9ldlL34Dj+29GA==</latexit> p " := ⌘ ↵ Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i Web <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> ↵ := hrS, rLi := krSk2 <latexit sha1_base64="M/RIeDTgA9+KlWxjKPyg62RBgcU=">AAACd3icfZDfShtBFMYnq/VP2tpoL71waFCk2rArolIQBHvhhVKLRgUnhLOTk83gzOwyMyuGJQ/h03hrH6OP0jsncQWN4oGBH993DmfOF2dSWBeG/yrBxOSHqemZ2erHT5/nvtTmF85smhuOTZ7K1FzEYFEKjU0nnMSLzCCoWOJ5fLU/9M+v0ViR6lPXz7ClINGiKzg4L7Vrawxk1gP6c5cyCTqRSJmGWAI9Wf9R0iEzI6ddq4eNcFT0NUQl1ElZx+35yibrpDxXqB2XYO1lFGauVYBxgkscVFluMQN+BQleetSg0LaK0VUDuuyVDu2mxj/t6Eh9PlGAsravYt+pwPXsuDcU3/RiNbbZdXdahdBZ7lDzx8XdXFKX0mFktCMMcif7HoAb4f9OeQ8McOeDrVbZL/THGTzyi35naMCl5nvBwCRK6IE/NmHrQ3qvEW6eGj35nKPxVF/D2UYj2mps/dms7+2Xic+QRfKNrJKIbJM9ckCOSZNwckvuyD35W/kfLAUrwepja1ApZ76SFxVED4XmwJs=</latexit> <latexit sha1_base64="w34K6vB5MtbA/uqgO9CL2ZFmMfs=">AAACY3icfZDRSltBEIY3p9pqtDXaghdSWBoKpZRwTgi2FAqCXngjRtpEwU3DnM0kLu7uOezOKQ3HPE1v6wP5AH2PbmIEjaUDCx///MPs/Gmulac4vqlET5aWnz5bWa2urT9/sVHb3Or6rHASOzLTmTtLwaNWFjukSONZ7hBMqvE0vdyf9k9/oPMqs99onGPPwMiqoZJAQerXtkWKBPzzFy6uhIVUA/8qrr43+7V63IhnxR9DMoc6m1e7v1lpiUEmC4OWpAbvz5M4p14JjpTUOKmKwmMO8hJGeB7QgkHfK2cXTPjboAz4MHPhWeIz9f5ECcb7sUmD0wBd+MXeVPxnLzULm2n4qVcqmxeEVt4uHhaaU8an8fCBcihJjwOAdCr8ncsLcCAphFitigMMxzk8CouOc3RAmXtfCnAjo+wkHDsSH6b0PyP8vDMGCjkni6k+hm6zkew2dk9a9b39eeIrbIe9Ye9Ywj6yPXbI2qzDJJuwX+w3u678idairejVrTWqzGdesgcVvf4LThq4xg==</latexit> progressive sharpening 大 用 3 2 0 氏 2 8 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS
  91. xt+1 ' <latexit sha1_base64="SfEqv5FaCw2PA3UwjePBBT0BA0A=">AAACa3icfZDfShtBFMYn22rT+C+pd23BwSCosWFXxPZSsBfeSBUaFdywnJ2cxMGZ2XXmbElYctmn6W37MH2IvkMnMQWN0gMDP77zHc6cL82VdBSGvyvBi5cLi6+qr2tLyyura/XGmwuXFVZgR2Qqs1cpOFTSYIckKbzKLYJOFV6mt8eT/uU3tE5m5iuNcuxqGBjZlwLIS0l9Y5iU1IrGPHZS4x3/sB21YiTgo4R2hgkl9WbYDqfFn0I0gyab1VnSqBzEvUwUGg0JBc5dR2FO3RIsSaFwXIsLhzmIWxjgtUcDGl23nF4y5lte6fF+Zv0zxKfqw4kStHMjnXqnBrpx872J+Gwv1XObqf+pW0qTF4RG3C/uF4pTxicx8Z60KEiNPICw0v+dixuwIMiHWavFn9EfZ/HUL/qSowXK7G4Zgx1oacb+2EG8N6H/GWH4z+jJ5xzNp/oULvbb0WH78PygeXQ8S7zK3rFNts0i9pEdsRN2xjpMsO/sB/vJflX+BOvB2+D9vTWozGbW2aMKtv4CCPK72w==</latexit> (1 + ⌘yt )xt ✓ <latexit

    sha1_base64="qSFFU1sZf3G4CopwKjmefxwqr14=">AAACi3icfZBda9RAFIZn41ddrW7rpTeDi1A/uiRraUUEC1XwRqzgtoXOGk5mT7JDZ5I4cyINIT/HX+OtXvhvnGxX0K14YODhPe/hzHmTUitHYfizF1y5eu36jbWb/Vu31+/cHWxsHrmishInstCFPUnAoVY5TkiRxpPSIphE43FydtD1j7+gdarIP1Jd4tRAlqtUSSAvxYNXddzQk6jdrmPiwimDn7lAAqExpS0BupwD3+Yi6bTUgmzOY/o0bsfCqmxOj+LBMByFi+KXIVrCkC3rMN7o7YhZISuDOUkNzp1GYUnTBiwpqbHti8phCfIMMjz1mINBN20Wl7b8oVdmPC2sfznxhfrnRAPGudok3mmA5m6114n/7CVmZTOlz6eNysuKMJcXi9NKcyp4FyOfKYuSdO0BpFX+71zOwedDPux+X7xGf5zFd37R+xItUGEfNwJsZlTe+mMz8bSj/xnh/LfRk885Wk31MhyNR9HuaPfDznD/YJn4GrvPHrAtFrE9ts/eskM2YZJ9Zd/Yd/YjWA+eBS+ClxfWoLecucf+quDNL4q7yJs=</latexit> yt ' ⌘ ↵ yt+1 ( [Damian+ <latexit sha1_base64="I08CI+X67LS8Tp0IIeEpq3Uc6XM=">AAACk3icfVBtaxNBEN6cb+350lTxk18WgyCi4S6UKohQ2oJ+KVYw7ZVuDHObucvSvb1jd08MR35Sf42fCvpfnCQnaCoOLDwz8zwzO09aaeV8FF11ghs3b92+s7EZ3r13/8FWd/vhiStrK3EoS13aJAWHWhkceuU1JpVFKFKNp+nFwaJ/+hWtU6X57GcVjgrIjcqUBE+lcfe9SDFXppE0w81DMSk9T/g7npxxIVbpGaUCdDWFVyKzIEnhYZB8GYQCzaRVjru9qB8tg18HcQt6rI3j8XZnh6bLukDjpQbnzuOo8qMGrFdSI32ldliBvIAczwkaKNCNmuXFc/6MKhOelZae8XxZ/VPRQOHcrEiJWYCfuvXeovjPXlqsbfbZm1GjTFV7NHK1OKs19yVf2MknyqL0ekYApFX0dy6nQC55Mj0MxSHScRaPaNHHCi340r5oBNi8UGZOx+bi5QL9jwjffhMJkc/xuqvXwcmgH+/2dz/t9PYOWsc32BP2lD1nMXvN9tgHdsyGTLJL9p39YD+Dx8HbYD84XFGDTqt5xP6K4OgXAsjKXg==</latexit> ◆ 2 xt 2   <latexit sha1_base64="epB8noL4M6eS8d8kUWCYUgUBfVw=">AAACnHicfVBdaxNBFJ2sXzV+NNVHQQaDUEXCrpS2CEKhfRCkGMGkkU5Y7k5utkNnZpeZu5Kwzd/yvwi+6u9wNo1iU/HCDIdzz+Xce7JSK09x/K0V3bh56/adjbvte/cfPNzsbD0a+qJyEgey0IUbZeBRK4sDUqRxVDoEk2k8yc4Pm/7JF3ReFfYTzUscG8itmioJFKi00xcZ5srWmQFyarYYbdMLIT43P9rJH5q/ecvXlBezlC6EmKd0RZl2unEvXha/DpIV6LJV9dOt1o6YFLIyaElq8P40iUsa1+BISY2Ltqg8liDPIcfTAC0Y9ON6efqCPw/MhE8LF54lvmT/nqjBeD83WVCGBc/8eq8h/9nLzJozTffHtbJlRWjlpfG00pwK3uTKJ8qhJD0PAKRTYXcuz8CBpJB+uy2OMBzn8DgYfSjRARXuZS3A5UbZRTg2F68a9D8hzH4LYZlzsp7qdTB83Ut2e7sfd7oHh6vEN9gT9oxts4TtsQP2jvXZgEn2lX1nP9jP6Gl0FL2Pji+lUWs185hdqWj4C1w/0do=</latexit> X(t) |xt | := Y (t) yt ( Ẋ = XY 2 Ẏ = ↵ 2 X <latexit sha1_base64="fdTY2ah/uzsGz/PaRkZMsycTMTI=">AAAClXicfVDfaxNBEN6cVdv4o6k++ODLYhBENNyFUn0pFFpEH0ormP7cGOY2c5elu3vH7p4YjvxN/jU+9EX/FSfJCZqKAwvfzHzfzM6Xllr5EMfXrejW2u07d9c32vfuP3i42dl6dOKLykkcyEIX7iwFj1pZHAQVNJ6VDsGkGk/Tq/15//QLOq8K+ylMSxwayK3KlIRApVHng0gxV7aWNMPP2mJcBH7Bd/k5F2KZnVMmQJcTeC0yB5IEAfr4ue5fEB/tuNGOOt24Fy+C3wRJA7qsiePRVmubFsjKoA1Sg/eXSVyGYQ0uKKmRhlceS5BXkOMlQQsG/bBe3Dzjz6ky5lnh6NnAF9U/FTUY76cmJaaBMPGrvXnxn73UrGwO2dthrWxZBbRyuTirNA8FnxvKx8qhDHpKAKRT9HcuJ0BGBbK93RYHSMc5PKRFRyU6CIV7WQtwuVF2Rsfm4tUc/Y8IX38TCZHPyaqrN8FJv5fs9HY+bnf39hvH19lT9oy9YAl7w/bYe3bMBkyyb+w7+8F+Rk+i3eggerekRq1G85j9FdHRL3vWy3s=</latexit> <latexit sha1_base64="QjsYPyjKNAYD2QgX6cNm5jf6Bp8=">AAACXXicfVBdSxtBFJ1sq7XxK2kffOjLYBBUJOwWUSkUBH3wpVTBaNAN4e7kZh2cj2Xmbtuw5Kf0tf1NfepfcRJTqLH0wMCZc8/l3nuyQklPcfyrFr14ubD4aul1fXlldW290Xxz5W3pBHaEVdZ1M/CopMEOSVLYLRyCzhReZ/cnk/r1F3ReWnNJowJ7GnIjh1IABanfaN5s0w7/8JGnyua8Gz79Ritux1Pw5ySZkRab4bzfrO2nAytKjYaEAu9vk7igXgWOpFA4rqelxwLEPeR4G6gBjb5XTXcf862gDPjQuvAM8an6d0cF2vuRzoJTA935+dpE/Gct03OTaXjUq6QpSkIjHgcPS8XJ8kkwfCAdClKjQEA4GXbn4g4cCArx1evpKYbjHH4Kgz4X6ICs261ScLmWZhyOzdO9CfufEb79MQYWck7mU31Ort63k4P2wcV+6/hklvgSe8c22TZL2CE7ZmfsnHWYYF/Zd/aD/az9jhailWjt0RrVZj1v2RNEGw9yCLV/</latexit> Z(t) := log X(t) Hamiltonian 1 2 ( H(Z, Y ) = 2 Y Ż = Y Ẏ = ↵ ↵Z + 4 e e 2 2Z ) <latexit sha1_base64="7C46C+jl/FgSJN9NA+G8G8h3LG4=">AAACkXicfVBdaxNBFJ2sXzV+NLWPfRkMQtUadkOo+iCEVqEgYgXTj3TScHdyNxk6O7vM3JWGJb/IX9PH6p9xNomgqXhh4HDuuZw5J861chSG17Xg1u07d++t3a8/ePjo8Xpj48mRyworsSczndmTGBxqZbBHijSe5BYhjTUexxf71f74G1qnMvOVpjkOUhgblSgJ5Klh44MgvCRHU41cpEATCZofbPd3Tp/zd1wkFmTUPj1v81cCdD4B3ucvF7SIkaCD52W7Pxs2mmErnA+/CaIlaLLlHA43ah0xymSRoiGpwbmzKMxpUIIlJTXO6qJwmIO8gDGeeWggRTco53ln/JlnRjzJrH+G+Jz986KE1LlpGntlFcmt7iryn7s4XXGm5M2gVCYvCI1cGCeF5pTxqkw+UhYl6akHIK3yf+dyAr4c8pXX6+I9+nAWP3mjzzlaoMy+KAXYcarMzIcdi50K/U8Il7+FHvmeo9VWb4Kjdivabe1+6TS7+8vG19gWe8q2WcResy47YIesxyT7zq7YD/Yz2AzeBt1gbyENasubTfbXBB9/AcNRyNc=</latexit> Hamiltonian 2Z {H = H0 } <latexit sha1_base64="V++YX7aEqjYwObb+c6oBqsvd0JQ=">AAACZ3icfZDdSgMxEIXT9b/+VQURvIkWQUTKroh6Iwj1ojeiglXBLWU2ndZgkl2SrFiWXvg03urj+Ai+hdlaUas4EPg4c8LMnCgR3Fjffy14I6Nj4xOTU8Xpmdm5+dLC4qWJU82wzmIR6+sIDAqusG65FXidaAQZCbyK7qp5/+oeteGxurDdBBsSOoq3OQPrpGZpNcxCCfaWgaC1wy9s+mGvWSr7Fb9f9DcEAyiTQZ01Fwq7YStmqURlmQBjbgI/sY0MtOVMYK8YpgYTYHfQwRuHCiSaRta/okc3nNKi7Vi7pyztq99/ZCCN6crIOfM1zXAvF//sRXJosm0fNDKuktSiYh+D26mgNqZ5RLTFNTIrug6Aae52p+wWNDDrgiwWw2N0x2k8cYNOE9RgY72VhaA7kqueO7YTbuf0nxEePo2OXM7BcKq/4XKnEuxV9s53y0fVQeKTZJWsk00SkH1yRGrkjNQJI4/kiTyTl8KbN+8teysfVq8w+LNEfpS39g7Ymrrq</latexit> Alex Damian Instability <latexit sha1_base64="Je3TUjQFY3Rxci9aQNkPwPfvzMU=">AAACbHicfZDfalNBEMY3p9XW+C9tvSvCYlBEJJwjpS1CoVAvvBErmLbQDWHOZpIs3X/dnVMMh9z6NN7qu/gSPoObNEJNxYGFH998w+x8pdcqUp7/bGQrq3furq3fa95/8PDR49bG5kl0VZDYlU67cFZCRK0sdkmRxjMfEEyp8bS8OJr1T68wROXsZ5p47BkYWTVUEihJ/RYXVxDQR6Wd5W8PuEACLuJlIC5A+zH0W+28k8+L34ZiAW22qOP+RmNHDJysDFqSGmI8L3JPvRoCKalx2hRVRA/yAkZ4ntCCwdir56dM+fOkDPjQhfQs8bl6c6IGE+PElMlpgMZxuTcT/9krzdJmGu73amV9RWjl9eJhpTk5PsuJD1RASXqSAGRQ6e9cjiGApJRmsyneYTou4Ie06KPHAOTCq1pAGBllp+nYkXg9o/8Z4csfY6KUc7Gc6m04edMpdju7n3bah0eLxNfZNnvGXrKC7bFD9p4dsy6T7Cv7xr6zH41f2ZNsO3t6bc0ai5kt9ldlL34Dj+29GA==</latexit> p " := ⌘ ↵ Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i Web <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> ↵ := hrS, rLi := krSk2 <latexit sha1_base64="M/RIeDTgA9+KlWxjKPyg62RBgcU=">AAACd3icfZDfShtBFMYnq/VP2tpoL71waFCk2rArolIQBHvhhVKLRgUnhLOTk83gzOwyMyuGJQ/h03hrH6OP0jsncQWN4oGBH993DmfOF2dSWBeG/yrBxOSHqemZ2erHT5/nvtTmF85smhuOTZ7K1FzEYFEKjU0nnMSLzCCoWOJ5fLU/9M+v0ViR6lPXz7ClINGiKzg4L7Vrawxk1gP6c5cyCTqRSJmGWAI9Wf9R0iEzI6ddq4eNcFT0NUQl1ElZx+35yibrpDxXqB2XYO1lFGauVYBxgkscVFluMQN+BQleetSg0LaK0VUDuuyVDu2mxj/t6Eh9PlGAsravYt+pwPXsuDcU3/RiNbbZdXdahdBZ7lDzx8XdXFKX0mFktCMMcif7HoAb4f9OeQ8McOeDrVbZL/THGTzyi35naMCl5nvBwCRK6IE/NmHrQ3qvEW6eGj35nKPxVF/D2UYj2mps/dms7+2Xic+QRfKNrJKIbJM9ckCOSZNwckvuyD35W/kfLAUrwepja1ApZ76SFxVED4XmwJs=</latexit> <latexit sha1_base64="w34K6vB5MtbA/uqgO9CL2ZFmMfs=">AAACY3icfZDRSltBEIY3p9pqtDXaghdSWBoKpZRwTgi2FAqCXngjRtpEwU3DnM0kLu7uOezOKQ3HPE1v6wP5AH2PbmIEjaUDCx///MPs/Gmulac4vqlET5aWnz5bWa2urT9/sVHb3Or6rHASOzLTmTtLwaNWFjukSONZ7hBMqvE0vdyf9k9/oPMqs99onGPPwMiqoZJAQerXtkWKBPzzFy6uhIVUA/8qrr43+7V63IhnxR9DMoc6m1e7v1lpiUEmC4OWpAbvz5M4p14JjpTUOKmKwmMO8hJGeB7QgkHfK2cXTPjboAz4MHPhWeIz9f5ECcb7sUmD0wBd+MXeVPxnLzULm2n4qVcqmxeEVt4uHhaaU8an8fCBcihJjwOAdCr8ncsLcCAphFitigMMxzk8CouOc3RAmXtfCnAjo+wkHDsSH6b0PyP8vDMGCjkni6k+hm6zkew2dk9a9b39eeIrbIe9Ye9Ywj6yPXbI2qzDJJuwX+w3u678idairejVrTWqzGdesgcVvf4LThq4xg==</latexit> progressive sharpening 大 用 面 3 2 0 氏 2 9 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 1 1 EoS ]
  92. xt+1 ' <latexit sha1_base64="SfEqv5FaCw2PA3UwjePBBT0BA0A=">AAACa3icfZDfShtBFMYn22rT+C+pd23BwSCosWFXxPZSsBfeSBUaFdywnJ2cxMGZ2XXmbElYctmn6W37MH2IvkMnMQWN0gMDP77zHc6cL82VdBSGvyvBi5cLi6+qr2tLyyura/XGmwuXFVZgR2Qqs1cpOFTSYIckKbzKLYJOFV6mt8eT/uU3tE5m5iuNcuxqGBjZlwLIS0l9Y5iU1IrGPHZS4x3/sB21YiTgo4R2hgkl9WbYDqfFn0I0gyab1VnSqBzEvUwUGg0JBc5dR2FO3RIsSaFwXIsLhzmIWxjgtUcDGl23nF4y5lte6fF+Zv0zxKfqw4kStHMjnXqnBrpx872J+Gwv1XObqf+pW0qTF4RG3C/uF4pTxicx8Z60KEiNPICw0v+dixuwIMiHWavFn9EfZ/HUL/qSowXK7G4Zgx1oacb+2EG8N6H/GWH4z+jJ5xzNp/oULvbb0WH78PygeXQ8S7zK3rFNts0i9pEdsRN2xjpMsO/sB/vJflX+BOvB2+D9vTWozGbW2aMKtv4CCPK72w==</latexit> (1 + ⌘yt )xt ✓ <latexit

    sha1_base64="qSFFU1sZf3G4CopwKjmefxwqr14=">AAACi3icfZBda9RAFIZn41ddrW7rpTeDi1A/uiRraUUEC1XwRqzgtoXOGk5mT7JDZ5I4cyINIT/HX+OtXvhvnGxX0K14YODhPe/hzHmTUitHYfizF1y5eu36jbWb/Vu31+/cHWxsHrmishInstCFPUnAoVY5TkiRxpPSIphE43FydtD1j7+gdarIP1Jd4tRAlqtUSSAvxYNXddzQk6jdrmPiwimDn7lAAqExpS0BupwD3+Yi6bTUgmzOY/o0bsfCqmxOj+LBMByFi+KXIVrCkC3rMN7o7YhZISuDOUkNzp1GYUnTBiwpqbHti8phCfIMMjz1mINBN20Wl7b8oVdmPC2sfznxhfrnRAPGudok3mmA5m6114n/7CVmZTOlz6eNysuKMJcXi9NKcyp4FyOfKYuSdO0BpFX+71zOwedDPux+X7xGf5zFd37R+xItUGEfNwJsZlTe+mMz8bSj/xnh/LfRk885Wk31MhyNR9HuaPfDznD/YJn4GrvPHrAtFrE9ts/eskM2YZJ9Zd/Yd/YjWA+eBS+ClxfWoLecucf+quDNL4q7yJs=</latexit> yt ' ⌘ ↵ yt+1 ( Taylor [Damian+ <latexit sha1_base64="I08CI+X67LS8Tp0IIeEpq3Uc6XM=">AAACk3icfVBtaxNBEN6cb+350lTxk18WgyCi4S6UKohQ2oJ+KVYw7ZVuDHObucvSvb1jd08MR35Sf42fCvpfnCQnaCoOLDwz8zwzO09aaeV8FF11ghs3b92+s7EZ3r13/8FWd/vhiStrK3EoS13aJAWHWhkceuU1JpVFKFKNp+nFwaJ/+hWtU6X57GcVjgrIjcqUBE+lcfe9SDFXppE0w81DMSk9T/g7npxxIVbpGaUCdDWFVyKzIEnhYZB8GYQCzaRVjru9qB8tg18HcQt6rI3j8XZnh6bLukDjpQbnzuOo8qMGrFdSI32ldliBvIAczwkaKNCNmuXFc/6MKhOelZae8XxZ/VPRQOHcrEiJWYCfuvXeovjPXlqsbfbZm1GjTFV7NHK1OKs19yVf2MknyqL0ekYApFX0dy6nQC55Mj0MxSHScRaPaNHHCi340r5oBNi8UGZOx+bi5QL9jwjffhMJkc/xuqvXwcmgH+/2dz/t9PYOWsc32BP2lD1nMXvN9tgHdsyGTLJL9p39YD+Dx8HbYD84XFGDTqt5xP6K4OgXAsjKXg==</latexit> ◆ 2 xt 2 w0 <latexit sha1_base64="1i4iQFRf8DBQBWM1roJuSwTbfOk=">AAACVXicfZDdSiNBEIV7Rtef6K5GL71pDAsiS5hZxPUyoBfeiApGhUwINZ2a2Ng/Q3ePGoY8hrf6WOLDCPbECBplCxo+Tp2mqk6aC25dFD0H4czsj7n5hcXa0vLPXyur9bVzqwvDsM200OYyBYuCK2w77gRe5gZBpgIv0uv9qn9xg8Zyrc7cMMeuhIHiGWfgvNRJJLirNKO3vai32oia0bjoV4gn0CCTOunVg52kr1khUTkmwNpOHOWuW4JxnAkc1ZLCYg7sGgbY8ahAou2W451H9LdX+jTTxj/l6Fj9+KMEae1Qpt5Z7Wine5X4bS+VU5NdttctucoLh4q9Dc4KQZ2mVSC0zw0yJ4YegBnud6fsCgww52Or1ZID9McZPPKDjnM04LTZLhMwA8nVyB87SP5U9D8j3L0bPfmc4+lUv8L532a829w93Wm09ieJL5ANskm2SEz+kRY5JCekTRjR5J48kMfgKXgJZ8O5N2sYTP6sk08VrrwCu5i1VA==</latexit>   <latexit sha1_base64="epB8noL4M6eS8d8kUWCYUgUBfVw=">AAACnHicfVBdaxNBFJ2sXzV+NNVHQQaDUEXCrpS2CEKhfRCkGMGkkU5Y7k5utkNnZpeZu5Kwzd/yvwi+6u9wNo1iU/HCDIdzz+Xce7JSK09x/K0V3bh56/adjbvte/cfPNzsbD0a+qJyEgey0IUbZeBRK4sDUqRxVDoEk2k8yc4Pm/7JF3ReFfYTzUscG8itmioJFKi00xcZ5srWmQFyarYYbdMLIT43P9rJH5q/ecvXlBezlC6EmKd0RZl2unEvXha/DpIV6LJV9dOt1o6YFLIyaElq8P40iUsa1+BISY2Ltqg8liDPIcfTAC0Y9ON6efqCPw/MhE8LF54lvmT/nqjBeD83WVCGBc/8eq8h/9nLzJozTffHtbJlRWjlpfG00pwK3uTKJ8qhJD0PAKRTYXcuz8CBpJB+uy2OMBzn8DgYfSjRARXuZS3A5UbZRTg2F68a9D8hzH4LYZlzsp7qdTB83Ut2e7sfd7oHh6vEN9gT9oxts4TtsQP2jvXZgEn2lX1nP9jP6Gl0FL2Pji+lUWs185hdqWj4C1w/0do=</latexit> X(t) |xt | := Y (t) yt Ẋ = XY 2 Ẏ = ↵ 2 X ( <latexit sha1_base64="fdTY2ah/uzsGz/PaRkZMsycTMTI=">AAAClXicfVDfaxNBEN6cVdv4o6k++ODLYhBENNyFUn0pFFpEH0ormP7cGOY2c5elu3vH7p4YjvxN/jU+9EX/FSfJCZqKAwvfzHzfzM6Xllr5EMfXrejW2u07d9c32vfuP3i42dl6dOKLykkcyEIX7iwFj1pZHAQVNJ6VDsGkGk/Tq/15//QLOq8K+ylMSxwayK3KlIRApVHng0gxV7aWNMPP2mJcBH7Bd/k5F2KZnVMmQJcTeC0yB5IEAfr4ue5fEB/tuNGOOt24Fy+C3wRJA7qsiePRVmubFsjKoA1Sg/eXSVyGYQ0uKKmRhlceS5BXkOMlQQsG/bBe3Dzjz6ky5lnh6NnAF9U/FTUY76cmJaaBMPGrvXnxn73UrGwO2dthrWxZBbRyuTirNA8FnxvKx8qhDHpKAKRT9HcuJ0BGBbK93RYHSMc5PKRFRyU6CIV7WQtwuVF2Rsfm4tUc/Y8IX38TCZHPyaqrN8FJv5fs9HY+bnf39hvH19lT9oy9YAl7w/bYe3bMBkyyb+w7+8F+Rk+i3eggerekRq1G85j9FdHRL3vWy3s=</latexit> <latexit sha1_base64="QjsYPyjKNAYD2QgX6cNm5jf6Bp8=">AAACXXicfVBdSxtBFJ1sq7XxK2kffOjLYBBUJOwWUSkUBH3wpVTBaNAN4e7kZh2cj2Xmbtuw5Kf0tf1NfepfcRJTqLH0wMCZc8/l3nuyQklPcfyrFr14ubD4aul1fXlldW290Xxz5W3pBHaEVdZ1M/CopMEOSVLYLRyCzhReZ/cnk/r1F3ReWnNJowJ7GnIjh1IABanfaN5s0w7/8JGnyua8Gz79Ritux1Pw5ySZkRab4bzfrO2nAytKjYaEAu9vk7igXgWOpFA4rqelxwLEPeR4G6gBjb5XTXcf862gDPjQuvAM8an6d0cF2vuRzoJTA935+dpE/Gct03OTaXjUq6QpSkIjHgcPS8XJ8kkwfCAdClKjQEA4GXbn4g4cCArx1evpKYbjHH4Kgz4X6ICs261ScLmWZhyOzdO9CfufEb79MQYWck7mU31Ort63k4P2wcV+6/hklvgSe8c22TZL2CE7ZmfsnHWYYF/Zd/aD/az9jhailWjt0RrVZj1v2RNEGw9yCLV/</latexit> Z(t) := log X(t) Hamiltonian 1 2 ( H(Z, Y ) = 2 Y Ż = Y Ẏ = ↵ ↵Z + 4 e e 2 2Z ) <latexit sha1_base64="7C46C+jl/FgSJN9NA+G8G8h3LG4=">AAACkXicfVBdaxNBFJ2sXzV+NLWPfRkMQtUadkOo+iCEVqEgYgXTj3TScHdyNxk6O7vM3JWGJb/IX9PH6p9xNomgqXhh4HDuuZw5J861chSG17Xg1u07d++t3a8/ePjo8Xpj48mRyworsSczndmTGBxqZbBHijSe5BYhjTUexxf71f74G1qnMvOVpjkOUhgblSgJ5Klh44MgvCRHU41cpEATCZofbPd3Tp/zd1wkFmTUPj1v81cCdD4B3ucvF7SIkaCD52W7Pxs2mmErnA+/CaIlaLLlHA43ah0xymSRoiGpwbmzKMxpUIIlJTXO6qJwmIO8gDGeeWggRTco53ln/JlnRjzJrH+G+Jz986KE1LlpGntlFcmt7iryn7s4XXGm5M2gVCYvCI1cGCeF5pTxqkw+UhYl6akHIK3yf+dyAr4c8pXX6+I9+nAWP3mjzzlaoMy+KAXYcarMzIcdi50K/U8Il7+FHvmeo9VWb4Kjdivabe1+6TS7+8vG19gWe8q2WcResy47YIesxyT7zq7YD/Yz2AzeBt1gbyENasubTfbXBB9/AcNRyNc=</latexit> Hamiltonian 2Z {H = H0 } <latexit sha1_base64="V++YX7aEqjYwObb+c6oBqsvd0JQ=">AAACZ3icfZDdSgMxEIXT9b/+VQURvIkWQUTKroh6Iwj1ojeiglXBLWU2ndZgkl2SrFiWXvg03urj+Ai+hdlaUas4EPg4c8LMnCgR3Fjffy14I6Nj4xOTU8Xpmdm5+dLC4qWJU82wzmIR6+sIDAqusG65FXidaAQZCbyK7qp5/+oeteGxurDdBBsSOoq3OQPrpGZpNcxCCfaWgaC1wy9s+mGvWSr7Fb9f9DcEAyiTQZ01Fwq7YStmqURlmQBjbgI/sY0MtOVMYK8YpgYTYHfQwRuHCiSaRta/okc3nNKi7Vi7pyztq99/ZCCN6crIOfM1zXAvF//sRXJosm0fNDKuktSiYh+D26mgNqZ5RLTFNTIrug6Aae52p+wWNDDrgiwWw2N0x2k8cYNOE9RgY72VhaA7kqueO7YTbuf0nxEePo2OXM7BcKq/4XKnEuxV9s53y0fVQeKTZJWsk00SkH1yRGrkjNQJI4/kiTyTl8KbN+8teysfVq8w+LNEfpS39g7Ymrrq</latexit> Alex Damian Instability <latexit sha1_base64="Je3TUjQFY3Rxci9aQNkPwPfvzMU=">AAACbHicfZDfalNBEMY3p9XW+C9tvSvCYlBEJJwjpS1CoVAvvBErmLbQDWHOZpIs3X/dnVMMh9z6NN7qu/gSPoObNEJNxYGFH998w+x8pdcqUp7/bGQrq3furq3fa95/8PDR49bG5kl0VZDYlU67cFZCRK0sdkmRxjMfEEyp8bS8OJr1T68wROXsZ5p47BkYWTVUEihJ/RYXVxDQR6Wd5W8PuEACLuJlIC5A+zH0W+28k8+L34ZiAW22qOP+RmNHDJysDFqSGmI8L3JPvRoCKalx2hRVRA/yAkZ4ntCCwdir56dM+fOkDPjQhfQs8bl6c6IGE+PElMlpgMZxuTcT/9krzdJmGu73amV9RWjl9eJhpTk5PsuJD1RASXqSAGRQ6e9cjiGApJRmsyneYTou4Ie06KPHAOTCq1pAGBllp+nYkXg9o/8Z4csfY6KUc7Gc6m04edMpdju7n3bah0eLxNfZNnvGXrKC7bFD9p4dsy6T7Cv7xr6zH41f2ZNsO3t6bc0ai5kt9ldlL34Dj+29GA==</latexit> p " := ⌘ ↵ Sharpening xt := hu, wt w0 i yt := hrS, wt w0 i Web <latexit sha1_base64="LfHZmm1V2nLhJKRQWvpx/Lh/QVY=">AAAChXicfZDfahNBFMYnq7Y12prWS28GgyDSht0SYlsQC/XCG7GCaQvdsJydnN0OnZldZs5qw5Jn8Wm81eu+TWeT+C8tHhj48Z1vOOd8aamkozC8bgX37j9YWV172H70eH3jSWdz68QVlRU4FIUq7FkKDpU0OCRJCs9Ki6BThafp5VHTP/2C1snCfKZJiSMNuZGZFEBeSjr7Vwnxgzc8VmByhTzWQBdpxqvt3/g1oZ0/HMZ27kw63bAXzorfhmgBXbao42Sz1Y/Hhag0GhIKnDuPwpJGNViSQuG0HVcOSxCXkOO5RwMa3aie3TjlL7wy5llh/TPEZ+rfP2rQzk106p3Nrm6514h39lK9NJmyvVEtTVkRGjEfnFWKU8GbAPlYWhSkJh5AWOl35+ICLAjyMbfb8Tv0x1n84Ad9LNECFfZVHYPNtTRTf2webzf0PyNc/TJ68jlHy6nehpPdXjToDT71u4dHi8TX2DP2nL1kEXvNDtl7dsyGTLBv7Dv7wX4Gq8FO0A8Gc2vQWvx5yv6p4O0NL13FTQ==</latexit> <latexit sha1_base64="DtN9lfiXckklduKUATiJuOsrT1g=">AAAChnicfVBdaxNBFJ2sVWP8aKqPvgwNgkgNu6UfUhAC9cEXaUtNW+iEcHdydzt0ZnaZuauGJf/FX+OrvvpvnE1SqKl4YODMuedy7z1pqZWnOP7diu6t3X/wsP2o8/jJ02fr3Y3nZ76onMShLHThLlLwqJXFISnSeFE6BJNqPE+vD5v6+Rd0XhX2M01LHBnIrcqUBArSuHswHRM/eM+FBptr5MJCqoGfbnFhgK7SjH8Nhre3frFwC+u424v78Rz8LkmWpMeWOB5vtHbEpJCVQUtSg/eXSVzSqAZHSmqcdUTlsQR5DTleBmrBoB/V8yNn/FVQJjwrXHiW+Fy93VGD8X5q0uBsdvWrtUb8Zy01K5MpezeqlS0rQisXg7NKcyp4kyCfKIeS9DQQkE6F3bm8AgeSQs6djviA4TiHn8KgoxIdUOHe1AJcbpSdhWNzsdWw/xnh240xsJBzsprqXXK23U/2+nsnO73B4TLxNnvJNtlrlrB9NmAf2TEbMsm+sx/sJ/sVtaN+tBvtL6xRa9nzgv2FaPAHtajFAg==</latexit> ↵ := hrS, rLi := krSk2 <latexit sha1_base64="M/RIeDTgA9+KlWxjKPyg62RBgcU=">AAACd3icfZDfShtBFMYnq/VP2tpoL71waFCk2rArolIQBHvhhVKLRgUnhLOTk83gzOwyMyuGJQ/h03hrH6OP0jsncQWN4oGBH993DmfOF2dSWBeG/yrBxOSHqemZ2erHT5/nvtTmF85smhuOTZ7K1FzEYFEKjU0nnMSLzCCoWOJ5fLU/9M+v0ViR6lPXz7ClINGiKzg4L7Vrawxk1gP6c5cyCTqRSJmGWAI9Wf9R0iEzI6ddq4eNcFT0NUQl1ElZx+35yibrpDxXqB2XYO1lFGauVYBxgkscVFluMQN+BQleetSg0LaK0VUDuuyVDu2mxj/t6Eh9PlGAsravYt+pwPXsuDcU3/RiNbbZdXdahdBZ7lDzx8XdXFKX0mFktCMMcif7HoAb4f9OeQ8McOeDrVbZL/THGTzyi35naMCl5nvBwCRK6IE/NmHrQ3qvEW6eGj35nKPxVF/D2UYj2mps/dms7+2Xic+QRfKNrJKIbJM9ckCOSZNwckvuyD35W/kfLAUrwepja1ApZ76SFxVED4XmwJs=</latexit> <latexit sha1_base64="w34K6vB5MtbA/uqgO9CL2ZFmMfs=">AAACY3icfZDRSltBEIY3p9pqtDXaghdSWBoKpZRwTgi2FAqCXngjRtpEwU3DnM0kLu7uOezOKQ3HPE1v6wP5AH2PbmIEjaUDCx///MPs/Gmulac4vqlET5aWnz5bWa2urT9/sVHb3Or6rHASOzLTmTtLwaNWFjukSONZ7hBMqvE0vdyf9k9/oPMqs99onGPPwMiqoZJAQerXtkWKBPzzFy6uhIVUA/8qrr43+7V63IhnxR9DMoc6m1e7v1lpiUEmC4OWpAbvz5M4p14JjpTUOKmKwmMO8hJGeB7QgkHfK2cXTPjboAz4MHPhWeIz9f5ECcb7sUmD0wBd+MXeVPxnLzULm2n4qVcqmxeEVt4uHhaaU8an8fCBcihJjwOAdCr8ncsLcCAphFitigMMxzk8CouOc3RAmXtfCnAjo+wkHDsSH6b0PyP8vDMGCjkni6k+hm6zkew2dk9a9b39eeIrbIe9Ye9Ywj6yPXbI2qzDJJuwX+w3u678idairejVrTWqzGdesgcVvf4LThq4xg==</latexit> progressive sharpening 大 用 面 3 2 0 氏 2 0 Damian, A., Nichani, E., & Lee, J. D. (2023). Self-stabilization: The implicit bias of gradient descent at the edge of stability. 2 1 EoS ]
  93. descent lemma (EoS) sharpness 2 sharpness む Open question? Progressive

    sharpening <latexit sha1_base64="AakLZ2nqfSWq5nnDy/G52PwrNR4=">AAACgHicfZDfShtBFMYnW2012jbqZW8GQyGRku5KsEUoCPaiF4qWNn8gm4azs2fj4MzsMjPbGpa8SJ+mt/YNfBtn415oUnpg4Md3vsOZ80WZ4Mb6/l3Ne7a2/vzFxmZ9a/vlq9eNnd2+SXPNsMdSkephBAYFV9iz3AocZhpBRgIH0fVp2R/8RG14qr7bWYZjCVPFE87AOmnS6H5rhRLsVZTQX236iYbCzcYwKZx6M2+FCiIBPw7p2SNbe9Jo+h1/UXQVggqapKrLyU6tG8YpyyUqywQYMwr8zI4L0JYzgfN6mBvMgF3DFEcOFUg042Jx3py+dUpMk1S7pyxdqI8nCpDGzGTknOUfzXKvFP/Zi+TSZpt8HBdcZblFxR4WJ7mgNqVldjTmGpkVMwfANHd/p+wKNDDrEq7Xw8/ojtN47hZdZKjBpvqgCEFPJVdzd+w0fFfS/4wu9MroyOUcLKe6Cv3DTnDUOfrabZ6cVolvkDdkn7RIQD6QE/KFXJIeYeQ3+UNuyV/P81reey94sHq1amaPPCnv+B6gp8LG</latexit> S(w) = 2 2 max (r L(w)) ⌘ <latexit sha1_base64="hGBKWI4k8xx9P+8t087oAwQx6Dw=">AAACTnicfZBNSwMxEIaz9avWbz16WSyCiJRdKepR0IMXUcGq4BaZTadtMMkuyaxYlv4Fr/qzvPpHvIlmawWt4kDg4Z03zMwbp1JYCoIXrzQ2PjE5VZ6uzMzOzS8sLi1f2CQzHBs8kYm5isGiFBobJEjiVWoQVCzxMr49KPqXd2isSPQ59VJsKuho0RYcqJAiJLhZrAa1YFD+bwiHUGXDOr1Z8upRK+GZQk1cgrXXYZBSMwdDgkvsV6LMYgr8Fjp47VCDQtvMB8v2/XWntPx2YtzT5A/U7z9yUNb2VOycCqhrR3uF+GcvViOTqb3XzIVOM0LNPwe3M+lT4hdJ+C1hkJPsOQBuhNvd510wwMnlValEh+iOM3jsBp2kaIASs5lHYDpK6L47thNtFfSfEe6/jI5czuFoqr/hYrsW7tR2zurV/YNh4mW2ytbYBgvZLttnR+yUNRhnXfbAHtmT9+y9em/e+6e15A3/rLAfVSp/AGYCtK4=</latexit> <latexit sha1_base64="TmyJuqC86/PYU9q7Oj7aBuESt84=">AAACYHicfZDPSiNBEMY74+4as39M1JN7aTYsLMsSZhZRj4IevIgKRgU7hJpOJWns7pntrhHjkHfxqm/k1SexE7OgcdmChh9ffU1VfWmulac4fqhEC+/ef1isLtU+fvr8ZbneWDn1WeEktmWmM3eegketLLZJkcbz3CGYVONZerk76Z9dofMqsyc0yrFjYGBVX0mgIHXrawIJhP/jSIDOh8CF1km33oxb8bT4W0hm0GSzOuo2Khuil8nCoCWpwfuLJM6pU4IjJTWOa6LwmIO8hAFeBLRg0HfK6fpj/j0oPd7PXHiW+FR9+aME4/3IpMFpgIZ+vjcR/9lLzdxk6m93SmXzgtDK58H9QnPK+CQb3lMOJelRAJBOhd25HIIDSSHBWk3sYTjO4UEYdJijA8rcz1KAGxhlx+HYgfg1of8Z4fqvMVDIOZlP9S2c/m4lm63N443mzu4s8Sr7yr6xHyxhW2yH7bMj1maS3bBbdsfuK49RNVqOGs/WqDL7s8peVbT+BKTFuI8=</latexit> p 見 ⌘ ↵⌧1 大 大 1 cf. self-stabilization 2 1 | GD
  94. ) vs. ( ) <latexit sha1_base64="2beUvCkyvZB/RdhsLSZwXSLFBHA=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVg/37ip1oNGMCj6G8Ih1MmwTm9q3k7USVkuUVkmwJjrMMhsuwBtORPYr0S5wQzYHXTx2qECiaZdDJbu0zWndGiSaveUpQP1+48CpDE9GTtnuaQZ7ZXin71Yjky2yX674CrLLSr2OTjJBbUpLROhHa6RWdFzAExztztlt6CBWZdbpRIdojtO47EbdJKhBpvqzSIC3ZVc9d2x3WirpP+M8PhldORyDkdT/Q0X241wt7F7tlNvHgwTnyYrZJWsk5DskSY5IqekRRjJyBN5Ji/eq/fuT/hTn1bfG/5ZJj/Kr34ADoa1dA==</latexit> w(T ) <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit>

    <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) w(0) 大 3 ( 2 1 ( ⌘  2/ ) <latexit sha1_base64="wzHh3HCDHQEVQunkPk/EINhiauc=">AAACXHicfZBNSwMxEIbT9bt+VQUvXoJFEJG6K6IeBT14ERWsCm4ps+m0BpPsksyKZek/8ar/yYu/xbRW0CoOBB7eecPMvEmmpKMwfCsFY+MTk1PTM+XZufmFxcrS8rVLcyuwLlKV2tsEHCppsE6SFN5mFkEnCm+Sh+N+/+YRrZOpuaJuhg0NHSPbUgB5qVmpxEjAY4V8dydOPDcr1bAWDor/hmgIVTasi+ZSaS9upSLXaEgocO4uCjNqFGBJCoW9cpw7zEA8QAfvPBrQ6BrFYPUe3/BKi7dT658hPlC//yhAO9fViXdqoHs32uuLf/YSPTKZ2oeNQposJzTic3A7V5xS3s+Ft6RFQarrAYSVfncu7sGCIJ9euRyfoD/O4pkfdJ6hBUrtVhGD7Whpev7YTrzdp/+M8PRl9ORzjkZT/Q3Xu7Vov7Z/uVc9Oh4mPs3W2DrbZBE7YEfslF2wOhPskT2zF/Zaeg/Gg9lg/tMalIZ/VtiPClY/AJVPtiA=</latexit> ( )
  95. : む <latexit sha1_base64="mHwqfM0W8YU5KmEzLdFd5TAEMio=">AAAClnicfZBtSxwxEMdz2wft2YezfVPwTehROIscu0W0byyClZai9Ao9Fcx1mc3NrsEkuyRZ9Vj2O/XTFHxVP0qz50nrWToQ+GXmP8zMPymksC4ML1vBvfsPHi4sPmovPX7y9Fln+fmBzUvDcchzmZujBCxKoXHohJN4VBgElUg8TE53mvrhGRorcv3NTQocKci0SAUH51Nx5/NejylwJ0landerW8yWKq7EVlR/1wyl7DEJOpNI/4jW6CQWN9+L2rOZSlbjTjfsh9OgdyGaQZfMYhAvt9bZOOelQu24BGuPo7BwowqME1xi3WalxQL4KWR47FGDQjuqpkfX9LXPjGmaG/+0o9Ps3x0VKGsnKvHKZlk7X2uS/6wlam6yS9+NKqGL0qHm14PTUlKX08ZROhYGuZMTD8CN8LtTfgIGuPO+t9vsA/rjDO77QV8KNOBy86ZiYDIldO2PzdhaQ/8TwsWN0JP3OZp39S4cvO1HG/2Nr+vd7Z2Z44tkhbwiPRKRTbJNPpEBGRJOfpCf5Be5Cl4G74Pd4OO1NGjNel6QWxEMfgNbvs5J</latexit> む L(w) = n X i=1

    <latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> 大 二 4 2 0 2 4 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
  96. : む <latexit sha1_base64="mHwqfM0W8YU5KmEzLdFd5TAEMio=">AAAClnicfZBtSxwxEMdz2wft2YezfVPwTehROIscu0W0byyClZai9Ao9Fcx1mc3NrsEkuyRZ9Vj2O/XTFHxVP0qz50nrWToQ+GXmP8zMPymksC4ML1vBvfsPHi4sPmovPX7y9Fln+fmBzUvDcchzmZujBCxKoXHohJN4VBgElUg8TE53mvrhGRorcv3NTQocKci0SAUH51Nx5/NejylwJ0landerW8yWKq7EVlR/1wyl7DEJOpNI/4jW6CQWN9+L2rOZSlbjTjfsh9OgdyGaQZfMYhAvt9bZOOelQu24BGuPo7BwowqME1xi3WalxQL4KWR47FGDQjuqpkfX9LXPjGmaG/+0o9Ps3x0VKGsnKvHKZlk7X2uS/6wlam6yS9+NKqGL0qHm14PTUlKX08ZROhYGuZMTD8CN8LtTfgIGuPO+t9vsA/rjDO77QV8KNOBy86ZiYDIldO2PzdhaQ/8TwsWN0JP3OZp39S4cvO1HG/2Nr+vd7Z2Z44tkhbwiPRKRTbJNPpEBGRJOfpCf5Be5Cl4G74Pd4OO1NGjNel6QWxEMfgNbvs5J</latexit> む L(w) = n X i=1

    <latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> : EoS Stage <latexit sha1_base64="B55qkJQFu/wxuKIQtEn5qJEykyM=">AAAClnicfVBdaxNBFJ2sXzV+NLUvBV8Gg5CKhl0p1RdLoYoiSiOYttCJy+zs3c2Qmdll5q5tWPY/+WsEn/SnOJumoKl4YeDcc8/lzjlJqaTDMPzRCa5dv3Hz1trt7p279+6v9zYeHLmisgLGolCFPUm4AyUNjFGigpPSAteJguNkdtDOj7+CdbIwn3FewkTz3MhMCo6einvvWWa5iJC5Ssf17FXYfKnxWdR8GDDNcZpk9VkTz7aZAsrOZAooVQr00PcZDi53rcynuB33+uEwXBS9CqIl6JNljeKNzg5LC1FpMCgUd+40Ckuc1NyiFAqaLqsclFzMeA6nHhquwU3qhemGPvZMSrPC+meQLtg/N2qunZvrxCtbJ2511pL/nCV65TJmLye1NGWFYMTF4axSFAvaJkpTaUGgmnvAhZX+71RMuU8Gfe7dLnsN3pyFj/7QYQmWY2Gf1IzbXEvTeLM5e9qi/wn5+aXQI59ztJrqVXD0fBjtDnc/7fT3D5aJr5GH5BEZkIi8IPvkHRmRMRHkG/lOfpJfwVawF7wJ3l5Ig85yZ5P8VcHoN/11zZQ=</latexit> 1 t t 1 X k=0 ✓ ◆ 1 e L(wk )  O t 大 二 4 2 0 1 2 5 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
  97. : む <latexit sha1_base64="mHwqfM0W8YU5KmEzLdFd5TAEMio=">AAAClnicfZBtSxwxEMdz2wft2YezfVPwTehROIscu0W0byyClZai9Ao9Fcx1mc3NrsEkuyRZ9Vj2O/XTFHxVP0qz50nrWToQ+GXmP8zMPymksC4ML1vBvfsPHi4sPmovPX7y9Fln+fmBzUvDcchzmZujBCxKoXHohJN4VBgElUg8TE53mvrhGRorcv3NTQocKci0SAUH51Nx5/NejylwJ0landerW8yWKq7EVlR/1wyl7DEJOpNI/4jW6CQWN9+L2rOZSlbjTjfsh9OgdyGaQZfMYhAvt9bZOOelQu24BGuPo7BwowqME1xi3WalxQL4KWR47FGDQjuqpkfX9LXPjGmaG/+0o9Ps3x0VKGsnKvHKZlk7X2uS/6wlam6yS9+NKqGL0qHm14PTUlKX08ZROhYGuZMTD8CN8LtTfgIGuPO+t9vsA/rjDO77QV8KNOBy86ZiYDIldO2PzdhaQ/8TwsWN0JP3OZp39S4cvO1HG/2Nr+vd7Z2Z44tkhbwiPRKRTbJNPpEBGRJOfpCf5Be5Cl4G74Pd4OO1NGjNel6QWxEMfgNbvs5J</latexit> む L(w) = n X i=1

    <latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> : EoS Stage <latexit sha1_base64="uVKnhPBYagm5kR5Bq1HUTeAdM2o=">AAACg3icfVDRahNBFJ2s1ta02lQffRkahFQl7IbQ9kUo1AcflFZI2kInhtnZu5uhM7PLzF1rWPZT+jV91Q/wb5xNU9BUvDBw7rnncuecuFDSYRj+agWPHq89Wd942t7cevZ8u7Pz4szlpRUwFrnK7UXMHShpYIwSFVwUFriOFZzHV8fN/PwbWCdzM8J5ARPNMyNTKTh6ato5+NRjmuMsTqvrejraYwoou5YJoFQJ0BPfp9hjqeUiqkZfBzWzMpvh3rTTDfvhouhDEC1BlyzrdLrTGrIkF6UGg0Jx5y6jsMBJxS1KoaBus9JBwcUVz+DSQ8M1uEm1cFjT155JaJpb/wzSBfvnRsW1c3Mde2Xjxq3OGvKfs1ivXMb0cFJJU5QIRtwdTktFMadNfDSRFgSquQdcWOn/TsWM+3TQh9xusw/gzVn47A+dFGA55vZNxbjNtDS1N5uxdw36n5B/vxd65HOOVlN9CM4G/Wi/v/9l2D06Xia+QV6RXdIjETkgR+QjOSVjIsgNuSU/yM9gLXgbDILhnTRoLXdekr8qeP8bawPFgg==</latexit> e L(wT )  O ✓ 1 T2 ◆ 見 大 二 4 2 0 2 2 6 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
  98. : む <latexit sha1_base64="mHwqfM0W8YU5KmEzLdFd5TAEMio=">AAAClnicfZBtSxwxEMdz2wft2YezfVPwTehROIscu0W0byyClZai9Ao9Fcx1mc3NrsEkuyRZ9Vj2O/XTFHxVP0qz50nrWToQ+GXmP8zMPymksC4ML1vBvfsPHi4sPmovPX7y9Fln+fmBzUvDcchzmZujBCxKoXHohJN4VBgElUg8TE53mvrhGRorcv3NTQocKci0SAUH51Nx5/NejylwJ0landerW8yWKq7EVlR/1wyl7DEJOpNI/4jW6CQWN9+L2rOZSlbjTjfsh9OgdyGaQZfMYhAvt9bZOOelQu24BGuPo7BwowqME1xi3WalxQL4KWR47FGDQjuqpkfX9LXPjGmaG/+0o9Ps3x0VKGsnKvHKZlk7X2uS/6wlam6yS9+NKqGL0qHm14PTUlKX08ZROhYGuZMTD8CN8LtTfgIGuPO+t9vsA/rjDO77QV8KNOBy86ZiYDIldO2PzdhaQ/8TwsWN0JP3OZp39S4cvO1HG/2Nr+vd7Z2Z44tkhbwiPRKRTbJNPpEBGRJOfpCf5Be5Cl4G74Pd4OO1NGjNel6QWxEMfgNbvs5J</latexit> む L(w) = n X i=1

    <latexit sha1_base64="syXruHIyJZx4DWoGWWlcpIJU6N0=">AAACXHicfZBNSwMxEIbT9avWr6rgxUuwCCpadkXUiyDowYtYwVbBrWU2nbbBJLsk2WJZ+k+86n/y4m8xrRW0igOBh3feMDNvlAhurO+/5byJyanpmfxsYW5+YXGpuLxSM3GqGVZZLGJ9F4FBwRVWLbcC7xKNICOBt9Hj2aB/20VteKxubC/BuoS24i3OwDqpUSyGKMSW3KYnFB+yPdlvFEt+2R8W/Q3BCEpkVJXGcu4gbMYslagsE2DMfeAntp6BtpwJ7BfC1GAC7BHaeO9QgURTz4ar9+mmU5q0FWv3lKVD9fuPDKQxPRk5pwTbMeO9gfhnL5Jjk23ruJ5xlaQWFfsc3EoFtTEd5EKbXCOzoucAmOZud8o6oIFZl16hEJ6jO07jpRt0laAGG+udLATdllz13bHtcHdA/xnh6cvoyOUcjKf6G2r75eCwfHh9UDo9GyWeJ+tkg2yRgByRU3JBKqRKGOmSZ/JCXnPv3qQ35y18Wr3c6M8q+VHe2gcyLrXt</latexit> `(m) = e む hw⇤ , yi xi i <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> <latexit sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> >0 `(hw, yi xi i) m <latexit sha1_base64="HVN1/W6stNVI4xeh2RL3r2DciNM=">AAACY3icfZDfShtBFMYnq60aa41/wAsRhgYhVg27IuqNIOiFN6JCo4KbhrOTk3Vw/iwzs2JY9mm81QfyAfoencQIbSw9MPDjO9/hzPmSTHDrwvC1EkxMfvo8NT1Tnf0y93W+trB4ZXVuGLaYFtrcJGBRcIUtx53Am8wgyETgdXJ/POhfP6CxXKsfrp9hW0KqeI8zcF7q1FZojEI05MZhLHTaiDbxZ7Ety41OrR42w2HRjxCNoE5GddFZqOzGXc1yicoxAdbeRmHm2gUYx5nAshrnFjNg95DirUcFEm27GF5Q0nWvdGlPG/+Uo0P1z4kCpLV9mXinBHdnx3sD8Z+9RI5tdr2DdsFVljtU7G1xLxfUaTqIh3a5QeZE3wMww/3fKbsDA8z5EKvV+AT9cQbP/KLzDA04bb4XMZhUclX6Y9N4a0D/M8Lju9GTzzkaT/UjXO00o73m3uVu/eh4lPg0WSXfSINEZJ8ckVNyQVqEkZI8kWfyUvkVzAaLwfKbNaiMZpbIXxWs/QaCurhe</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt `(m) = log(1 + e m ) ⌘rL(wt ) ; <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws )  min ⌘>0 <latexit sha1_base64="VJVFbPAq1jx6rJvWLoecIV5GI7c=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfRJBH3wpVTBR8ILMbSZxze7esTsnDUe+g6/2Y/XNb+Jbu4kR2igOLPvjP/9lZv9ZoaTjOH4MajOzcx/mFxbDj0ufPi+vrH5puby0gpoiV7m9zNCRkoaaLFnRZWEJdaboIusfjfoXd2SdzM05Dwpqa+wZ2ZUC2UutlBgP4uuVetyIxxW9hmQCdZjU6fVqsJN2clFqMiwUOneVxAW3K7QshaJhmJaOChR97NGVR4OaXLsarzuMvnqlE3Vz64/haKz++6JC7dxAZ96pkW/cdG8kvtnL9NRk7u63K2mKksmI58HdUkWcR6Msoo60JFgNPKCw0u8eiRu0KNgnFobpMfnPWfruB/0oyCLndqtK0fa0NEP/2V76bUTvGfHni9GTzzmZTvU1tLYbyW5j92ynfng0SXwB1mEDNiGBPTiEEziFJgi4hXt4gF/B7+Ap+FMLnq0vN6zBf1UL/wJke7Qx</latexit> ⇢ 1 `(0) , C⌘ n 大 二 4 立 2 0 2 7 Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large Stepsize Gradient Descent for Logistic Loss: Non-Monotonicity of the Loss Improves Optimization Efficiency. 2 1 📄 [Wu+ ]
  99. 0 <latexit sha1_base64="61VZmbWLM+jIBUwK0Mci8UMGDU0=">AAACgHicfZDbThsxEIad7YmGQ0N72RurESKc0l0U0apXSOGCm6pUagCJTaNZZ7JY2N6VPVsRVnkRnobb9g36NvWGIEFAHcny539mNJ4/yZV0FIZ/a8Gz5y9evlp4XV9cWl5501h9e+yywgrsiUxl9jQBh0oa7JEkhae5RdCJwpPkolvlT36hdTIzP2icY19DauRICiAvDRqdGJVqXW3ECvkULze2qnvdQ+tqx7+6gzhBAp7eKT93B41m2A6nwR9DNIMmm8XRYLXWiYeZKDQaEgqcO4vCnPolWJJC4aQeFw5zEBeQ4plHAxpdv5yuN+FrXhnyUWb9McSn6v2OErRzY534Sg107uZzlfhkLtFzk2n0uV9KkxeERtwOHhWKU8Yr7/hQWhSkxh5AWOn/zsU5WBDkHa7X4wP0y1n86gd9y9ECZXazjMGmWpqJXzaNtyv6XyFc3hV68j5H864+huPddrTX3vveae53Z44vsPfsA2uxiH1i++yQHbEeE+ya3bDf7E8QBK3gYxDdlga1Wc879iCCL/8AD/zBeQ==</latexit> `(z)  `(x) + ` (x)(z x)

    + C g(x)(z <latexit sha1_base64="H+h1+/KmO5QovvDz342jQP0z548=">AAACbnicfZDRShtBFIYnq1Yb2xor9KZIB4MgRZfdItYLLwS96E2pBaOCG8LZyUkcnJldZs6WxDW59mm8bV/Ft+gjdBIjaJQeGPj4zz+cc/40V9JRFN1VgpnZuVfzC6+ri2/evluqLb8/cVlhBTZEpjJ7loJDJQ02SJLCs9wi6FThaXp5MOqf/kLrZGaOqZ9jU0PXyI4UQF5q1daSTmZBKX612eMJYY/K4dCFFA6Hg+urrd71Xtyq1aMwGhd/DvEE6mxSR63lynbSzkSh0ZBQ4Nx5HOXULMGSFAoH1aRwmIO4hC6eezSg0TXL8TEDvu6VNvdb+WeIj9XHP0rQzvV16p0a6MJN90bii71UT02mzm6zlCYvCI24H9wpFKeMj5LibWlRkOp7AGGl352LC7AgyOdZrSaH6I+z+N0P+pGjBcrs5zIB29XSDPyx3WRzRP8zQu/B6MnnHE+n+hxOvoTxTrjzc7u+fzBJfIF9ZGtsg8XsK9tn39gRazDBbtgt+83+VP4GH4LV4NO9NahM/qywJxVs/AOVy76K</latexit> x) 8z, x s.t. |z 2 (⟹ x| < 1 0 `(z)  `(x) + ` (x)(z x) + 2 (z x) <latexit sha1_base64="OnCTTKGVPu/Z2b9XYVfIWMGd4BQ=">AAACY3icfZDdSiNBEIU7o7tqdNf4A17IQrNhwV0kzIiol0K88EZU2KjghFDTqYyN3T1Dd40YhzyNt/pAPoDvYSdG0LhsQcPHqVNU10lyJR2F4VMlmJr+8nVmdq46v/Dt+2JtafnMZYUV2BKZyuxFAg6VNNgiSQovcougE4XnyXVz2D+/QetkZv5SP8e2htTInhRAXurU1tKNu988VsibnThBghiV8lKnVg8b4aj4Z4jGUGfjOuksVbbjbiYKjYaEAucuozCndgmWpFA4qMaFwxzENaR46dGARtcuRxcM+C+vdHkvs/4Z4iP1/UQJ2rm+TrxTA125yd5Q/Gcv0RObqbfXLqXJC0IjXhf3CsUp48N4eFdaFKT6HkBY6f/OxRVYEORDrFbjA/THWTzyi45ztECZ/VPGYFMtzcAfm8abQ/qfEW7fjJ58ztFkqp/hbKsR7TR2Trfr+81x4rNsnf1kGyxiu2yfHbIT1mKCDdg9e2CPledgPlgOVl+tQWU8s8I+VPDjBQ7XuKc=</latexit> g(z)  C `(z) ) g(·) := |`0 (·)| per-sample loss gradient <latexit sha1_base64="lcEyyIKdACMSutjHiGMPtGiGuss=">AAACZnicfZDfShtBFMYnW21ttBpbSqG9GQylViTsFrGlUBDsRW9EBaOCG8LZyclmcP4sM2dLwxro0/S2fZ2+QR+jk7iCjdIDAz++8x3OnC8rlPQUx78b0YOFxYePlh43l1eerK611p+eels6gV1hlXXnGXhU0mCXJCk8LxyCzhSeZZf70/7ZV3ReWnNC4wJ7GnIjh1IABanfeplvpmJg6S3/+IlfpajUm1q46rfacSeeFb8LSQ1tVtdRf72xkw6sKDUaEgq8v0jignoVOJJC4aSZlh4LEJeQ40VAAxp9r5odMeGvgzLgQ+vCM8Rn6u2JCrT3Y50FpwYa+fneVLy3l+m5zTT80KukKUpCI64XD0vFyfJpQnwgHQpS4wAgnAx/52IEDgSFHJvN9DOG4xwehEWHBTog67aqFFyupZmEY/N0e0r/M8K3G2OgkHMyn+pdOH3XSXY7u8c77b39OvEl9optsE2WsPdsj31hR6zLBPvOfrCf7FfjT7QaPY9eXFujRj3zjP1TEf8LTaK5rA==</latexit> <latexit sha1_base64="wuva5hFDWFr8ZE5tKU8cW4NLSbE=">AAAChHicfZDfTxNBEMe3JyIWfxR99GVjQyyKzV1F5MmQ4IMvREgokHC1mdvOlQ27e5fdOUO53L/iX8MrvvvfuHfURItxks1+9jszmZ1vkivpKAx/toJ7S/eXH6w8bK8+evzkaWft2bHLCitwKDKV2dMEHCppcEiSFJ7mFkEnCk+Si706f/INrZOZOaJZjiMNUyNTKYC8NO7sxKhU72ojVsgbvNx4U9+vPPSu3tav1IIo4wQJqnJQ8Ub9Ohh3umE/bILfhWgOXTaPg/FaayueZKLQaEgocO4sCnMalWBJCoVVOy4c5iAuYIpnHg1odKOyWbHi616Z8DSz/hjijfpnRwnauZlOfKUGOneLuVr8Zy7RC5Mp3RmV0uQFoRG3g9NCccp47R+fSIuC1MwDCCv937k4B28ReZfb7fgT+uUs7vtBX3K0QJl9XcZgp1qayi87jTdr+l8hXP4u9OR9jhZdvQvHg3603d8+3Oru7s0dX2Ev2EvWYxH7wHbZZ3bAhkyw7+ya3bAfwXKwGbwL3t+WBq15z3P2VwQffwFRisPp</latexit> 2 む む Self-bdd. <latexit sha1_base64="0QlDEcyTvnyeSdugGqeYPFv5uw4=">AAACUnicfVJNa9tAEF25bZqqSZqkx15ETcAtwUjBuDkGkkMvpS7UiSEyZrQaK4v3Q+yOSozwn8i1+Vm99K/01JXjQmuXDCw83rzhzTw2K6VwFMc/g9aTp8+2nm+/CF/u7O692j84vHSmshyH3EhjRxk4lELjkARJHJUWQWUSr7LZedO/+obWCaO/0rzEsYJCi6ngQJ4aFZ2U54beTfbbcTdeVrQJkhVos1UNJgdBL80NrxRq4hKcu07iksY1WBJc4iJMK4cl8BkUeO2hBoVuXC8XXkRHnsmjqbH+aYqW7N8TNSjn5irzSgV049Z7DfnfXqbWnGl6Oq6FLitCzR+Mp5WMyERNGlEuLHKScw+AW+F3j/gNWODkMwvD9AL9cRY/eaPPJVogY9/XKdhCCb3wxxbpcYMeE8LtH6FHPudkPdVNcHnSTfrd/pde++x8lfg2e8Pesg5L2Ad2xj6yARsyziS7Y9/ZffAj+NXyv+RB2gpWM6/ZP9Xa+Q3yA7QA</latexit> g(·) ⌘  1/C g(x) <latexit sha1_base64="0zKR7c+UbJCVRlI2d3E5vsyJRrE=">AAACZXicfZDfShtBFMYnW+uftLaxiiC9cGgQVErcLaK9FOKFN6KFRgU3hLOTk3VwZnaZOVsMS/BpvK3P4xP0NTobI9hYPDDwm+98w5nzJbmSjsLwoRa8mXk7Oze/UH/3fvHDx8bSpzOXFVZgR2QqsxcJOFTSYIckKbzILYJOFJ4n1+2qf/4LrZOZ+UnDHLsaUiMHUgB5qddYi5GAxwp5tFO2e3FSXdPNm61Rr9EMW+G4+EuIJtBkkzrtLdV2434mCo2GhALnLqMwp24JlqRQOKrHhcMcxDWkeOnRgEbXLcc7jPiGV/p8kFl/DPGx+vxFCdq5oU68UwNdueleJf63l+ipyTT43i2lyQtCIx4HDwrFKeNVQLwvLQpSQw8grPR/5+IKLAjyMdbr8SH65Swe+0EnOVqgzG6XMdhUSzPyy6bx14peM8LNk9GTzzmaTvUlnH1rRXutvR+7zYP2JPF59pl9YZssYvvsgB2xU9Zhgt2yO/ab3df+BIvBSrD6aA1qkzfL7J8K1v8CrG25Yw==</latexit> 二 立 大 一 二 大 小 8 descent lemma 2 1 | Self-bounding property
  100. u = u1 + u 2 <latexit sha1_base64="UEzaDO+dTAq5eLZsZHqcGWjqb5c=">AAACcXicfZDNSgMxEMfT9bt+VT2Jl2ARRKXsiqgXQdCDF1HBquCWMpvOtsEkuyRZsSx9Ap/Gqz6Jz+ELmK1VtIoDgV/+M8PM/KNUcGN9/7XkjYyOjU9MTpWnZ2bn5isLi1cmyTTDOktEom8iMCi4wrrlVuBNqhFkJPA6ujsq8tf3qA1P1KXtptiQ0FY85gysk5qVtVCC7UQxzegB/eJmQDe//bablapf8/tBf0MwgCoZxHlzobQTthKWSVSWCTDmNvBT28hBW84E9sphZjAFdgdtvHWoQKJp5P17enTNKS0aJ9o9ZWlf/d6RgzSmKyNXWexohnOF+GcukkOTbbzfyLlKM4uKfQyOM0FtQguzaItrZFZ0HQDT3O1OWQc0MOssLZfDY3THaTx1g85S1GATvZGHoNuSq547th1uFfRfITx8FjpyPgfDrv6Gq+1asFvbvdipHh4NHJ8kK2SVrJOA7JFDckLOSZ0w8kieyDN5Kb15yx71Vj9KvdKgZ4n8CG/zHWUyvcg=</latexit> u /

    w⇤ <latexit sha1_base64="JNc9LqsQcokMM6+B9jobIILE/zk=">AAACaXicfVBdSxtBFJ1stWrUNmlfiiIMDYKIhN0i6qNgH/oiKhgV3BDuTu7GwfliZlYNS578Nb62v6a/oX/C2RjBRvHChcO553LvOZkR3Pk4/luLPszMfpybX6gvLi1/+txofjlzurAMO0wLbS8ycCi4wo7nXuCFsQgyE3ieXR9U8/MbtI5rdeqHBrsSBornnIEPVK+xlkrwV1lOC5oaq43X9Jm57W32Gq24HY+LvgbJBLTIpI57zdp22teskKg8E+DcZRIb3y3Bes4Ejupp4dAAu4YBXgaoQKLrlmMfI7oemD7NtQ2tPB2zLzdKkM4NZRaU1Y9uelaRb84yOXXZ53vdkitTeFTs6XBeCBrcVyHRPrfIvBgGAMzy8DtlV2CB+RBlvZ7+xGDO4mE4dGTQgtd2s0zBDiRXo2B2kG5V6D0h3D0LAwo5J9OpvgZnP9rJTnvnZLu1fzBJfJ6sku9kgyRkl+yTX+SYdAgj9+SB/CZ/av+iZvQtWnmSRrXJzlfyX0WtR9Wpu9c=</latexit> <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt <latexit sha1_base64="bwrIt4mhQRPcOefPV68YjElvXw0=">AAAC9XicfVFdTxQxFO0MKjh+sOCjL40bExTYzGwQfRFJ8MEHjZi4QELXzZ3unaGh05m0HWEz7E/xjZD45O/x39jZXcy4EG/S5PSec3ra27iQwtgw/O35C3fu3ltcuh88ePjo8XJrZfXA5KXm2OO5zPVRDAalUNizwko8KjRCFks8jE/3av7wO2ojcvXVjgrsZ5AqkQgO1rUGrZ/sgmVgT+KEng0qux6NN6/3Jbv41g02aVNhqds3efqWdhlaoEyCSiVSpiCWQD+uNUwvNv56Gn5HMD01rdP6DHeay7rNXycNWu2wE06K3gTRDLTJrPYHK94WG+a8zFBZLsGY4ygsbL8CbQWXOA5YabAAfgopHjuoIEPTryYzHdPnrjOkSa7dUpZOuk1HBZkxoyx2yvqiZp6rm7dycTaXbJM3/UqoorSo+DQ4KSW1Oa0/jA6FRm7lyAHgWri7U34CGrh13xoE7D26x2n85II+F6jB5vplxUCnmVBj99iUbdTof0I4vxY65OYczU/1JjjodqLtzvaXrfbu3mziS+QpeUbWSERek13ygeyTHuHesvfK2/He+Wf+D//Sv5pKfW/meUL+Kf/XH3Hd66o=</latexit> kwt+1 uk 2 ⌘rL(wt ) 2 kwt uk = 2⌘hrL(wt ), u 2 wt i + ⌘ krL(wt )k 2 2 <latexit sha1_base64="U56KgnYgjGsLibTfpZvw12g0ojk=">AAAC73ichVHLbtQwFHXCqx1eU1iysRghFShDElWlLJAqlQULEEVi2kr1NLrx3KRWbSfYDjBK5zvYISRWfBJ/gzMzFe20gitZOj73nvvMKimsi6LfQXjl6rXrN5aWOzdv3b5zt7tyb9eWteE44KUszX4GFqXQOHDCSdyvDILKJO5lx9utf+8zGitK/dGNKxwqKLTIBQfnqbT78xVNGDqgTIIuJFKmIZNA364yBe4oy+mX1D1eO/3UaUyf0TMuZmayp+fT/C9L8lfXyg4Tyk4u1bCTwyTt9qJ+NDV6EcRz0CNz20lXgnU2KnmtUDsuwdqDOKrcsAHjBJc46bDaYgX8GAo88FCDQjtsptuc0EeeGdG8NP5pR6fsWUUDytqxynxk26hd9LXkpb5MLVR2+eawEbqqHWo+K5zXkrqStqeiI2GQOzn2ALgRvnfKj8AAd/6gnQ57jX44g+98ofcVGnCledIwMIUSeuKHLdhai/4VCF9PAz3ye44Xt3oR7Cb9eKO/8WG9t7U93/gSeUAeklUSkxdki7whO2RAeLAcPA82g5fhp/Bb+D38MQsNg7nmPjln4a8/6X7pSQ==</latexit> = 2⌘hrL(wt ), u1 wt i + 2⌘hrL(wt ), u2 i + ⌘ krL 2 2 n X 1 = 2⌘hrL(w ), u w i + 2⌘hrL(w ), u i + ⌘ krL(w )k 0 t 1 t t 2 t = ` (hwt , zi i)hzi , u wt i <latexit sha1_base64="U56KgnYgjGsLibTfpZvw12g0ojk=">AAAC73ichVHLbtQwFHXCqx1eU1iysRghFShDElWlLJAqlQULEEVi2kr1NLrx3KRWbSfYDjBK5zvYISRWfBJ/gzMzFe20gitZOj73nvvMKimsi6LfQXjl6rXrN5aWOzdv3b5zt7tyb9eWteE44KUszX4GFqXQOHDCSdyvDILKJO5lx9utf+8zGitK/dGNKxwqKLTIBQfnqbT78xVNGDqgTIIuJFKmIZNA364yBe4oy+mX1D1eO/3UaUyf0TMuZmayp+fT/C9L8lfXyg4Tyk4u1bCTwyTt9qJ+NDV6EcRz0CNz20lXgnU2KnmtUDsuwdqDOKrcsAHjBJc46bDaYgX8GAo88FCDQjtsptuc0EeeGdG8NP5pR6fsWUUDytqxynxk26hd9LXkpb5MLVR2+eawEbqqHWo+K5zXkrqStqeiI2GQOzn2ALgRvnfKj8AAd/6gnQ57jX44g+98ofcVGnCledIwMIUSeuKHLdhai/4VCF9PAz3ye44Xt3oR7Cb9eKO/8WG9t7U93/gSeUAeklUSkxdki7whO2RAeLAcPA82g5fhp/Bb+D38MQsNg7nmPjln4a8/6X7pSQ==</latexit> <latexit sha1_base64="BGgV/BWIxgjg5aT1gN895LBz0QY=">AAADQHichVJNb9QwEHXCVwkf3cKRi8WKUtB2laCqcKlUqRy4IIrEthXrEDneSdaq7US2AyzRSlzhZ/Ev+AfcEBInTjjbFC27Kxgp8su8N3rjGael4MaG4VfPv3Dx0uUra1eDa9dv3FzvbNw6MkWlGQxYIQp9klIDgisYWG4FnJQaqEwFHKenBw1//Ba04YV6ZSclxJLmimecUetSSefn3ibJNGWRIqaSSc33oukbRUCI+1tEUJULwERSO04z/C6xvXP8IeFEz+gHizJH9f78VNtz1W0FJiTYJM25wllAZoeN/5J9tcocb+N58f9a1Twf2zjpdMN+OAu8DKIWdFEbh8mGt0NGBaskKMsENWYYhaWNa6otZwKmAakMlJSd0hyGDioqwcT1bD1TfM9lRjgrtPuUxbPsfEVNpTETmTpl07JZ5JrkSi6VC842exLXXJWVBcXOjLNKYFvgZvd4xDUwKyYOUKa56x2zMXU7sO6FBAF5Cu5yGp47oxclaGoL/bAmVOeSq6m7bE56DfqXkL4/Fzrk5hwtTnUZHD3qR7v93Zc73f2DduJr6A66i7ZQhB6jffQMHaIBYt5r76P3yfvsf/G/+d/9H2dS32trbqO/wv/1G7bNDaI=</latexit> n i=1 n X 1  [`(hu, zi i) n i=1 (convexity) `(hwt , zi i)] u2  L(u1 ) L(wt )  `( ✓) L(wt ) <latexit sha1_base64="9ZWOyn0nyc/PyHfOKCZdfA0cPFs=">AAACb3icfZDfShtBFMYnq7WattY/F14IMhgKptSwK6K9FPSiF4oWGhXcEM5OzsbBmdll5qw2LHkAn6a39VF8DN+gszGCxuKBgR/f+YZzzpfkSjoKw/taMDX9bub97Fz9w8dP858XFpdOXVZYgW2RqcyeJ+BQSYNtkqTwPLcIOlF4llztV/2za7ROZuYXDXLsaOgbmUoB5KXuQiNWyA83Yg10maS86EZNvvlMuOlS07vCVjgq/hqiMTTYuE66i7XtuJeJQqMhocC5iyjMqVOCJSkUDutx4TAHcQV9vPBoQKPrlKNrhvyLV3o8zax/hvhIff6jBO3cQCfeWS3pJnuV+N9eoicmU/q9U0qTF4RGPA5OC8Up41VUvCctClIDDyCs9LtzcQkWBPlA6/X4AP1xFo/8oOMcLVBmv5Yx2L6WZuiP7cffKnrLCL+fjJ58ztFkqq/hdKsV7bR2fm439vbHic+yVbbONljEdtke+8FOWJsJdsv+sL/srvYQrARrAX+0BrXxn2X2ooLmP+rdvKE=</latexit>  `(✓ ) <latexit sha1_base64="h1qMM3lLiQ9zHMfh2x2djBbtOw8=">AAACYXicfZDfShtBFMYn29bG+C8qvfJmMQhWJOwWsV4G7IU3xRQaFZwQzk5ONkNmZpeZs8Ww5GF6q0/Ua1/ESYygifTAwI/vfMM550tyJR1F0b9K8OHjp5XP1dXa2vrG5lZ9e+fKZYUV2BGZyuxNAg6VNNghSQpvcougE4XXyeh82r/+g9bJzPymcY5dDamRAymAvNSrf+EKOSp1yGmIBDwFreFrr96ImtGswmWI59Bg82r3tisnvJ+JQqMhocC52zjKqVuCJSkUTmq8cJiDGEGKtx4NaHTdcrb/JDzwSj8cZNY/Q+FMff2jBO3cWCfeqYGGbrE3Fd/tJXphMg3OuqU0eUFoxPPgQaFCysJpOGFfWhSkxh5AWOl3D8UQLAjyEdZq/Af64yz+9IMuc7RAmT0qOdhUSzPxx6b8eEr/M8Ldi9GTzzleTHUZrr4149Pm6a+TRut8nniV7bF9dshi9p212AVrsw4TrGR/2T17qDwGq0E92Hm2BpX5n132poK9J1HzuN4=</latexit> u1 = ✓w⇤ 非 9 <latexit sha1_base64="367e1GafQeTNBLWm/Mbt2TkJF4o=">AAACa3icfZDRShtBFIYna201rTXqnS04NAgiEnaLWG8EQS+8kVowKrhhOTs5mwzOzC4zZ9uGJZc+jbftw/gQvoOzMUIbSw8MfPznH845f1oo6SgM7xvB3Kv5128WFptv3y29X26trF64vLQCuyJXub1KwaGSBrskSeFVYRF0qvAyvTmq+5ff0TqZm3MaFdjTMDAykwLIS0lrI9ZAwzTjZRLxAx7TEAmetR/JdtJqh51wUvwlRFNos2mdJSuN3bifi1KjIaHAuesoLKhXgSUpFI6bcemwAHEDA7z2aECj61WTS8Z80yt9nuXWP0N8ov75owLt3Ein3lnv6GZ7tfjPXqpnJlO236ukKUpCI54GZ6XilPM6Jt6XFgWpkQcQVvrduRiCBUE+zGYzPkZ/nMVTP+hrgRYot9tVDHagpRn7YwfxTk3/M8LPZ6Mnn3M0m+pLuPjcifY6e99224dH08QX2Af2iW2xiH1hh+yEnbEuE+yW3bFf7HfjIVgL1oOPT9agMf2zxv6qYPMRpHW8Kg==</latexit> 2 1 | <latexit sha1_base64="4zh6H38M2fSQj2yfPP5OtPykIK8=">AAACc3icfZDPbhMxEMad5U9LCjSFYy9WU0SKaLRbVYVjpXLooRVFIm2lOopmndmNVdu7smeBaJVH4Gm4woPwINzrTYMEKWIkSz9981kz86WlVp7i+Gcrunf/wcOV1UfttcdPnq53Np6d+6JyEgey0IW7TMGjVhYHpEjjZekQTKrxIr0+avoXn9B5VdiPNC1xaCC3KlMSKEijzkuhkQvUuidyMAYETZBgh+/yk54wQJM0459HtDPqdON+PC9+F5IFdNmizkYbrX0xLmRl0JLU4P1VEpc0rMGRkhpnbVF5LEFeQ45XAS0Y9MN6ftGMvwjKmGeFC88Sn6t//qjBeD81aXA2S/rlXiP+s5eapcmUvR3WypYVoZW3g7NKcyp4ExcfK4eS9DQASKfC7lxOwIGkEGq7Ld5hOM7haRj0vkQHVLhXtQCXG2Vn4dhcvG7of0b48tsYKOScLKd6F873+slB/+DDfvfwaJH4KttkW6zHEvaGHbJjdsYGTLKv7Bv7zn60fkWb0Va0fWuNWos/z9lfFe3eAOZsvvQ=</latexit> (t ) 0
  101. u / w⇤ <latexit sha1_base64="JNc9LqsQcokMM6+B9jobIILE/zk=">AAACaXicfVBdSxtBFJ1stWrUNmlfiiIMDYKIhN0i6qNgH/oiKhgV3BDuTu7GwfliZlYNS578Nb62v6a/oX/C2RjBRvHChcO553LvOZkR3Pk4/luLPszMfpybX6gvLi1/+txofjlzurAMO0wLbS8ycCi4wo7nXuCFsQgyE3ieXR9U8/MbtI5rdeqHBrsSBornnIEPVK+xlkrwV1lOC5oaq43X9Jm57W32Gq24HY+LvgbJBLTIpI57zdp22teskKg8E+DcZRIb3y3Bes4Ejupp4dAAu4YBXgaoQKLrlmMfI7oemD7NtQ2tPB2zLzdKkM4NZRaU1Y9uelaRb84yOXXZ53vdkitTeFTs6XBeCBrcVyHRPrfIvBgGAMzy8DtlV2CB+RBlvZ7+xGDO4mE4dGTQgtd2s0zBDiRXo2B2kG5V6D0h3D0LAwo5J9OpvgZnP9rJTnvnZLu1fzBJfJ6sku9kgyRkl+yTX+SYdAgj9+SB/CZ/av+iZvQtWnmSRrXJzlfyX0WtR9Wpu9c=</latexit> <latexit sha1_base64="unho3cxyTiuKod1bYXhZ9eJI6zk=">AAACunicfVFNb9NAEF27BYr5aApHLisipBRoZEdV4cChUjlwAFEk0lbKmmi9GTur7K6t3TEQufmXXPgrnFinqZSmiJFWevPmjWbmbVYp6TCOfwfh1vadu/d27kcPHj56vNvZe3LmytoKGIpSlfYi4w6UNDBEiQouKgtcZwrOs9lJWz//DtbJ0nzFeQWp5oWRuRQcPTXuGHbJNMdpltMf4wZfJYuD67xml98G0QFdVyD1+Y06U0AHDJCPGCjVYwXXmjOcembfiz/21pr303GnG/fjZdDbIFmBLlnF6XgvOGSTUtQaDArFnRslcYVpwy1KoWARsdpBxcWMFzDy0HANLm2WxizoC89MaF5a/wzSJbve0XDt3FxnXtlu6TZrLfnPWqY3JmP+Nm2kqWoEI64G57WiWNLWdTqRFgSquQdcWOl3p2LKLRfo/yaK2Hvwx1n45Ad9rsByLO3LhnFbaGkW/tiCvW7R/4T857XQI+9zsunqbXA26CdH/aMvh93jk5XjO+QZeU56JCFvyDH5QE7JkAjyi/wJtoLt8F2YhTKcXUnDYNXzlNyIEP8CxrDXQg==</latexit> kwt+1 uk telescoping

    2 2 kwt uk  2⌘[`( ✓) <latexit sha1_base64="fwyVbnlH7TkuXCoxPOjCnkE+a9A=">AAAC2HicfVHfaxNBEN47f9X4K9UXwZe1QUjVhrtQqi9CoT74oFjBtMVuGuY2c8mS3btjd04N1wPfRPDJ/85/w7/ATZpgTcWBhY+Zb/ab+SYptHIURT+D8NLlK1evrV1v3Lh56/ad5vrdA5eXVmJP5jq3Rwk41CrDHinSeFRYBJNoPEwme7P64Ue0TuXZe5oW2DcwylSqJJBPDZrfRWpBVuJUGKBxkvJPA9pa4lKcnnTrqiuQgFPNn/A5OybhSjOoJi+i+qSirbh+3f7TPtnkQiMXqHVbjMAYEDT2H2wu28+JrQgMmq2oE82DXwTxArTYIvYH68G2GOayNJiR1ODccRwV1K/AkpIa64YoHRYgJzDCYw8zMOj61dy2mj/ymSFPc+tfRnyePd9RgXFuahLPnM3rVmuz5D9riVlRpvR5v1JZURJm8kw4LTWnnM9uwofKoiQ99QCkVX52LsfgrSJ/uUZDvES/nMU3XuhtgRYot48rAXZkVFb7ZUfi6Qz9jwifl0SPvM/xqqsXwUG3E+90dt5tt3b3Fo6vsQdsg7VZzJ6xXfaK7bMek+xXcD94GGyEH8Iv4dfw2xk1DBY999hfEf74DSu25EI=</latexit> 2 kwt uk 1 + 2⌘t t ( t 1 X k=0 L(wt )] 2 kuk L(wk )  `( ✓) + 2⌘t log t . t kwt k = O(log t) P t 1 1 L(w ) = Õ k k=0 t 大 1 1 0 Stage <latexit sha1_base64="+xD+NVBHLnh/1eLu46mwSiADlvs=">AAACZXicfVBdSxtBFJ2sWtP4FT8QxIcOBkFFwm6R6IsQSB98ERUaE3BDuDu5GwdnZpeZ2bZhCf4aX+3v6S/o3+hsjNBG6YGBM+eey733RKngxvr+r5I3N7/wYbH8sbK0vLK6Vl3fuDVJphm2WSIS3Y3AoOAK25Zbgd1UI8hIYCd6aBX1zjfUhifqqx2l2JMwVDzmDKyT+tWdg1CCvY9i+r3v03P6+vMP+9WaX/cnoG9JMCU1MsV1f710Eg4SlklUlgkw5i7wU9vLQVvOBI4rYWYwBfYAQ7xzVIFE08snN4zpvlMGNE60e8rSifp3Rw7SmJGMnLNY0czWCvHdWiRnJtv4rJdzlWYWFXsZHGeC2oQWAdEB18isGDkCTHO3O2X3oIFZF2OlEn5Bd5zGSzfoKkUNNtFHeQh6KLkau2OH4XHB/meEH69Gx1zOwWyqb8nt53rQqDduTmrN1jTxMtkle+SABOSUNMkFuSZtwsgjeSLP5Gfpt7fibXnbL1avNO3ZJP/A+/QH65i4/g==</latexit> (w0 = 0) <latexit sha1_base64="4LNZ+r0u5rnK1HXTzmotKVXLqro=">AAACY3icfZDfahNBFMYnq9Y2sTWtghdSGBoKpUjYlVK9LNQLb4oVTFvIhnB2cnY7dP4sM2fFsOzTeKsP1Afoezj5I2giHhj48Z3vcOZ8Wamkpzi+b0WPHj/ZeLq51e4829553t3du/K2cgIHwirrbjLwqKTBAUlSeFM6BJ0pvM7uzmf966/ovLTmC01LHGkojMylAArSuPsqVei9l5qnuQNRp8oWnBoad3txP54XX4dkCT22rMvxbusknVhRaTQkFHg/TOKSRjU4kkJh004rjyWIOyhwGNCARj+q5xc0/DAoE55bF54hPlf/nKhBez/VWXBqoFu/2puJ/+xlemUz5e9HtTRlRWjEYnFeKU6Wz+LhE+lQkJoGAOFk+DsXtxCSoRBiu51+wHCcw4uw6FOJDsi64zoFV2hpmnBskb6Z0f+M8O23MVDIOVlNdR2u3vaT0/7p55Pe2fky8U32mh2wI5awd+yMfWSXbMAEa9h39oP9bD1EnWgvermwRq3lzAv2V0X7vwDnYroc</latexit> <latexit sha1_base64="wvewJvL4pRbPk4N61k1H2T1teoc=">AAACynicfVFdb9MwFHXC1wgf6zbeeLGokFoEVYKmsZdJk8YDD0MdEt0mzSVy3JvUquNE9g1QZeGJP8k7PwSnywR0iCtZOrr3HF+f46RU0mIY/vD8W7fv3L23cT948PDR483e1vapLSojYCIKVZjzhFtQUsMEJSo4Lw3wPFFwliyO2vnZZzBWFvojLkuY5jzTMpWCo2vFvW8sgUzqWrg7bBOwS5ZznCcp/RIju6QHdDxgqsgoDiljAUsNFxEyW+VxvTgIm081voqa48Fv1WLoRJShVDOgY6YgxcG1zMhsjsOAgZ51C+NePxyFq6I3QdSBPunqJN7ydtmsEFUOGoXi1l5EYYnTmhuUQoFzUFkouVjwDC4c1DwHO61XQTX0uevMaFoYdzTSVfdPRc1za5d54pitIbs+a5v/nCX52mZM96e11GWFoMXV4rRSFAva/gKdSQMC1dIBLox0b6dizl1K6P4qCNhbcOYMvHeLxiUYjoV5UTNuslzqxpnN2MsW/Y/Iv14THXI5R+up3gSnr0fR3mjvw27/8KhLfIM8Jc/IgETkDTkk78gJmRBBfnqBt+M98Y994y/9+orqe51mh/xV/vdff2ndnQ==</latexit> 3 1 | 1 t Stage : gradient factor η 2∥∇L(wt)∥2 ✓ <latexit sha1_base64="VgYy5a5oIqjiPy5psH9PUslRGGQ=">AAACUHicfZBNSyNBEIZr4rq64374cfQyGBZkkTAj4noU9OBFdMFEwQlS06kkvXb3DN01YhjyH7zqz/LmP/GmnRhB47IFTT+89TZV/WaFko7j+CGozXya/Tw3/yVc+Prt+4/FpeWWy0srqClylduzDB0paajJkhWdFZZQZ4pOs8u9Uf/0iqyTuTnhQUFtjT0ju1Ige6mVcp8YLxbrcSMeV/QRkgnUYVLHF0vBVtrJRanJsFDo3HkSF9yu0LIUioZhWjoqUFxij849GtTk2tV43WH00yudqJtbfwxHY/Xtiwq1cwOdeadG7rvp3kj8Zy/TU5O5u9OupClKJiNeBndLFXEejbKIOtKSYDXwgMJKv3sk+mhRsE8sDNN98p+zdOgHHRVkkXP7q0rR9rQ0Q//ZXroxov8Z8frV6MnnnEyn+hFam41ku7H9Z6u+uzdJfB5WYQ3WIYHfsAsHcAxNEPAXbuAW7oL74DF4qgUv1tcbVuBd1cJnNiK0nw==</latexit> (t )
  102. u / w⇤ <latexit sha1_base64="JNc9LqsQcokMM6+B9jobIILE/zk=">AAACaXicfVBdSxtBFJ1stWrUNmlfiiIMDYKIhN0i6qNgH/oiKhgV3BDuTu7GwfliZlYNS578Nb62v6a/oX/C2RjBRvHChcO553LvOZkR3Pk4/luLPszMfpybX6gvLi1/+txofjlzurAMO0wLbS8ycCi4wo7nXuCFsQgyE3ieXR9U8/MbtI5rdeqHBrsSBornnIEPVK+xlkrwV1lOC5oaq43X9Jm57W32Gq24HY+LvgbJBLTIpI57zdp22teskKg8E+DcZRIb3y3Bes4Ejupp4dAAu4YBXgaoQKLrlmMfI7oemD7NtQ2tPB2zLzdKkM4NZRaU1Y9uelaRb84yOXXZ53vdkitTeFTs6XBeCBrcVyHRPrfIvBgGAMzy8DtlV2CB+RBlvZ7+xGDO4mE4dGTQgtd2s0zBDiRXo2B2kG5V6D0h3D0LAwo5J9OpvgZnP9rJTnvnZLu1fzBJfJ6sku9kgyRkl+yTX+SYdAgj9+SB/CZ/av+iZvQtWnmSRrXJzlfyX0WtR9Wpu9c=</latexit> <latexit sha1_base64="unho3cxyTiuKod1bYXhZ9eJI6zk=">AAACunicfVFNb9NAEF27BYr5aApHLisipBRoZEdV4cChUjlwAFEk0lbKmmi9GTur7K6t3TEQufmXXPgrnFinqZSmiJFWevPmjWbmbVYp6TCOfwfh1vadu/d27kcPHj56vNvZe3LmytoKGIpSlfYi4w6UNDBEiQouKgtcZwrOs9lJWz//DtbJ0nzFeQWp5oWRuRQcPTXuGHbJNMdpltMf4wZfJYuD67xml98G0QFdVyD1+Y06U0AHDJCPGCjVYwXXmjOcembfiz/21pr303GnG/fjZdDbIFmBLlnF6XgvOGSTUtQaDArFnRslcYVpwy1KoWARsdpBxcWMFzDy0HANLm2WxizoC89MaF5a/wzSJbve0XDt3FxnXtlu6TZrLfnPWqY3JmP+Nm2kqWoEI64G57WiWNLWdTqRFgSquQdcWOl3p2LKLRfo/yaK2Hvwx1n45Ad9rsByLO3LhnFbaGkW/tiCvW7R/4T857XQI+9zsunqbXA26CdH/aMvh93jk5XjO+QZeU56JCFvyDH5QE7JkAjyi/wJtoLt8F2YhTKcXUnDYNXzlNyIEP8CxrDXQg==</latexit> kwt+1 uk 2

    ( <latexit sha1_base64="wvewJvL4pRbPk4N61k1H2T1teoc=">AAACynicfVFdb9MwFHXC1wgf6zbeeLGokFoEVYKmsZdJk8YDD0MdEt0mzSVy3JvUquNE9g1QZeGJP8k7PwSnywR0iCtZOrr3HF+f46RU0mIY/vD8W7fv3L23cT948PDR483e1vapLSojYCIKVZjzhFtQUsMEJSo4Lw3wPFFwliyO2vnZZzBWFvojLkuY5jzTMpWCo2vFvW8sgUzqWrg7bBOwS5ZznCcp/RIju6QHdDxgqsgoDiljAUsNFxEyW+VxvTgIm081voqa48Fv1WLoRJShVDOgY6YgxcG1zMhsjsOAgZ51C+NePxyFq6I3QdSBPunqJN7ydtmsEFUOGoXi1l5EYYnTmhuUQoFzUFkouVjwDC4c1DwHO61XQTX0uevMaFoYdzTSVfdPRc1za5d54pitIbs+a5v/nCX52mZM96e11GWFoMXV4rRSFAva/gKdSQMC1dIBLox0b6dizl1K6P4qCNhbcOYMvHeLxiUYjoV5UTNuslzqxpnN2MsW/Y/Iv14THXI5R+up3gSnr0fR3mjvw27/8KhLfIM8Jc/IgETkDTkk78gJmRBBfnqBt+M98Y994y/9+orqe51mh/xV/vdff2ndnQ==</latexit> kwt 2 uk  2⌘[`( ✓) kwt k = O(log t) P t 1 1 L(w ) = Õ k k=0 t L(wt )] 1 t t=⌧ <latexit sha1_base64="I2jv2AR5B9/yt60k24FIDiP3320=">AAACUHicfZBRSxtBEMfnola92qr10ZfDUBAp4U5EfREEffClVMFEwQsyt5nENbt7x+6cNBz5Dr7aj9U3v4lv7SZGaKM4sOyP//yXmf1nhZKO4/gxqM3Mzn2YX1gMPy59+ry8svql5fLSCmqKXOX2MkNHShpqsmRFl4Ul1Jmii6x/NOpf3JF1MjfnPCiorbFnZFcKZC+1+CBlLK9X6nEjHlf0GpIJ1GFSp9erwU7ayUWpybBQ6NxVEhfcrtCyFIqGYVo6KlD0sUdXHg1qcu1qvO4w+uqVTtTNrT+Go7H674sKtXMDnXmnRr5x072R+GYv01OTubvfrqQpSiYjngd3SxVxHo2yiDrSkmA18IDCSr97JG7QomCfWBimx+Q/Z+m7H/SjIIuc260qRdvT0gz9Z3vptxG9Z8SfL0ZPPudkOtXX0NpuJLuN3bOd+uHRJPEFWIcN2IQE9uAQTuAUmiDgFu7hAX4Fv4On4E8teLa+3LAG/1Ut/AsCSLSE</latexit> <latexit sha1_base64="cViaMrcYFJb2M8rn+UuFdFPBMqM=">AAACuXicfVHfaxNBEN5c/VHjr1QffVkMQioa7kqohVIoxAcflFYwbaGbHnObuWTN7t2xO2cNx/2VPvmn+OZeElFTcWCXj2++mdn5Nim0chSG31vB1q3bd+5u32vff/Dw0ePOzpMzl5dW4kjmOrcXCTjUKsMRKdJ4UVgEk2g8T+bDJn/+Ba1TefaJFgWODUwzlSoJ5Km4Y0RqQUaCoOTClSau5kdhfVU1xOuoft8TBmiWpPw6nu/yIy6u1QRJ6QnyE6Expd7vBsKq6Yx2uef5iq0Gw1gkSHC1J/xdx51u2A+XwW+CaA26bB2n8U5rICa5LA1mJDU4dxmFBY0rsKSkxrotSocFyDlM8dLDDAy6cbX0peYvPDPhaW79yYgv2T8rKjDOLUzilc2WbjPXkP/MJWZjMqUH40plRUmYydXgtNScct6YzifKoiS98ACkVf7tXM7AO0T+a9pt8Rb9chY/+EEnBVqg3L6sBNipUVntl52KVw36nxC+/hJ65H2ONl29Cc72+tF+f//joHs8XDu+zZ6x56zHIvaGHbN37JSNmGTf2I9W0NoKDgMIZsHnlTRorWuesr8icD8BMMTYhA==</latexit> ⌧ 1 X k=0 ✓ ◆ 1 1 e L(wk ) = O  2 ⌧ 4C ⌘ <latexit sha1_base64="u3lxIOGR0i3u3O20skfSBRJYWsk=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hoa6FbZDadrqFJdkmyYln6E7zqz/LsD/EmYvohaBUHAg/vvGFm3jDhTBvPe3VyM7Nz8/mFxcLS8srq2npxo6HjVFGs05jHqhmCRs4k1g0zHJuJQhAhx5uwVxv2b+5RaRbLa9NPsC0gkqzLKBgrXQUcb9dLXtkblfsb/AmUyKQubotOJejENBUoDeWgdcv3EtPOQBlGOQ4KQaoxAdqDCFsWJQjU7Wy068DdsUrH7cbKPmnckfr9RwZC674IrVOAudPTvaH4Zy8UU5NN97idMZmkBiUdD+6m3DWxOwzC7TCF1PC+BaCK2d1degcKqLFxFQrBCdrjFJ7ZQecJKjCx2ssCUJFgcmCPjYL9If1nhIcvoyWbsz+d6m9oHJT9w/LhZaVUrU0SXyBbZJvsEp8ckSo5JRekTiiJyCN5Is/Oi/PmvDsfY2vOmfzZJD8ql/8Ed7u0Ow==</latexit>  1 ⌧ <latexit sha1_base64="2ljverZQLX+GqePeABCI3IMelrg=">AAACWHicfZDdSiNBEIUrs64/0fX30pvGIOiyhBkR9VJwL7xQVNio4MRQ06lJGrt7hu4ed8OQ9/BW30qfxp6YBTcuFjR8nDpNVZ0kl8K6MHyuBV+mvk7PzM7V5xe+LS4tr6xe2qwwnFo8k5m5TtCSFJpaTjhJ17khVImkq+TuqOpf3ZOxItO/3CCntsKeFqng6Lx0e7IVK3T9JGW/O3a7s9wIm+Go2EeIxtCAcZ13Vmq7cTfjhSLtuERrb6Iwd+0SjRNc0rAeF5Zy5HfYoxuPGhXZdjlae8g2vdJlaWb8046N1Pc/SlTWDlTindWSdrJXif/tJWpisksP2qXQeeFI87fBaSGZy1iVCesKQ9zJgQfkRvjdGe+jQe58cvV6/JP8cYZO/aCznAy6zHwvYzQ9JfTQH9uLf1T0mRH//DV68jlHk6l+hMudZrTX3LvYbRwejROfhXXYgC2IYB8O4RjOoQUcDDzAIzzVXgIIZoK5N2tQG/9Zg38qWH0Fuwa1Uw==</latexit> 大 1 L(ws ) 3 1 十 |
  103. <latexit sha1_base64="w2KNB8ex1F9thBmvzfwkZfYLohI=">AAACb3icfVBdaxNBFJ1sq7bxox8++FAog0FoRdLdElofC/GhDxUrmLbQieHu5G4ydGZ2mblrDUt+gL/GV/tT+jP8B86mETQVL8xwOPdczr0nLbTyFMe3jWhp+cHDRyurzcdPnj5bW9/YPPN56ST2ZK5zd5GCR60s9kiRxovCIZhU43l61a3751/QeZXbTzQpsG9gZFWmJFCgBuutkx1hgMZpxq8HfpcLjTzZ63QHIkWCz/si/EEVt+NZ8fsgmYMWm9fpYKPREcNclgYtSQ3eXyZxQf0KHCmpcdoUpccC5BWM8DJACwZ9v5pdM+WvAjPkWe7Cs8Rn7J8TFRjvJyYNynpzv9iryX/2UrPgTNnbfqVsURJaeWeclZpTzuuo+FA5lKQnAYB0KuzO5RgcSAqBNpviHYbjHL4PRh8KdEC5e10JcCOj7DQcOxJvavQ/IXz9LQwo5JwspnofnO23k4P2wcdO66g7T3yFbbGXbIcl7JAdsWN2ynpMsm/sO/vBbho/oxfRdsTvpFFjPvOc/VXR7i/Qyr0X</latexit> t=s ) L(wt+1 )  L(wt ) <latexit

    sha1_base64="ma7DEH8yS3YCblisYhSz4SzZvxg=">AAACn3icfVHbbhMxEHW2XEq4pfDIiyFCaoFGuyVqeaxUJHjgEgRpi+oQzTqzqVXbu7JngWi7H8an8MQr/AXeNEglRYxk6ejMGZ2Z47TQylMcf29FK5cuX7m6eq19/cbNW7c7a3f2fV46iUOZ69wdpuBRK4tDUqTxsHAIJtV4kJ7sNf2Dz+i8yu0HmhU4MjC1KlMSKFDjzvtX68IAHacZ/zKu6HFSb3ChkZ+naYNvCsocyOqpQIK66tdcnAoLqYYlpTj9tDXudONePC9+ESQL0GWLGozXWn0xyWVp0JLU4P1REhc0qsCRkhrrtig9FiBPYIpHAVow6EfV/PqaPwzMhGe5C88Sn7PnJyow3s9MGpTNon6515D/7KVmyZmyZ6NK2aIktPLMOCs1p5w30fKJcihJzwIA6VTYnctjCLFR+IB2WzzHcJzD18HobYEOKHePKgFuapStw7FT8aRB/xPC1z/CgELOyXKqF8H+Vi/Z7m2/63d39xaJr7J77AFbZwnbYbvsJRuwIZPsG/vBfrJf0f3oRfQmGpxJo9Zi5i77q6KPvwE4hM8j</latexit> G(wt )  1/4C ⌘ <latexit sha1_base64="Yhb1pcehKIiJ2QBW9IYKNuMU7EM=">AAACbXicfVBdaxNBFJ2sVWvqR1rxSZHRIFaRuFtC9bEQwb6ILZi20AnL3cnddOjM7DJzVw1Lnv01fW1/S3+Ff8HZNIKm4oULh3PP5dx7slIrT3F82YpurNy8dXv1Tnvt7r37DzrrGwe+qJzEoSx04Y4y8KiVxSEp0nhUOgSTaTzMTgfN/PArOq8K+4WmJY4MTKzKlQQKVNp59nFTGKCTLOffUnrFhUaevO0PUpEhgQiddrpxL54Xvw6SBeiyRe2l662+GBeyMmhJavD+OIlLGtXgSEmNs7aoPJYgT2GCxwFaMOhH9fyXGX8RmDHPCxfaEp+zf27UYLyfmiwom7v98qwh/znLzJIz5e9HtbJlRWjllXFeaU4Fb4LiY+VQkp4GANKpcDuXJ+BAUoiz3RYfMDzn8FMw+lyiAyrc61qAmxhlZ+HZiXjToP8J4ftvYUAh52Q51evgYKuXbPe29/vdncEi8VX2mD1nmyxh79gO22V7bMgk+8HO2Dm7aP2MHkVPoqdX0qi12HnI/qro5S9V2bxv</latexit> descent lemma 3⌘ 2 4 krL(wt )k <latexit sha1_base64="q04lpq/46hkCWO5p5byeZ6PfZps=">AAAClnicfZBvSxtBEMY3V9va9I/RvhF8szQUYpFwV8QWQRGsKIViCo0KbnrMbebOxd29Y3evNh75Tv00BV/Zj9K9GIuNpQMLP+Z5lpl5kkIK68LwqhE8mHv46PH8k+bTZ89fLLQWl45sXhqOfZ7L3JwkYFEKjX0nnMSTwiCoROJxcr5b68ff0FiR6y9uVOBAQaZFKjg434pbH/c7TIE7S1J6sUo3tyhLDfBIM1uquBJb0firzjpMgs4k0j/WtVu6jAUzE3E1brXDbjgpeh+iKbTJtHrxYmOdDXNeKtSOS7D2NAoLN6jAOMEljpustFgAP4cMTz1qUGgH1eToMX3tO0Oa5sY/7eike/dHBcrakUq8s97Vzmp1859aomYmu/T9oBK6KB1qfjM4LSV1Oa0TpUNhkDs58gDcCL875WfgU3Q+92aTfUB/nMFPftBhgQZcbt5UDEymhB77YzO2VtP/jPD91ujJ5xzNpnofjt52o43uxuf19s7uNPF5skJekQ6JyDuyQw5Ij/QJJz/IT3JNfgXLwXawF+zfWIPG9M9L8lcFvd8cN8w8</latexit> G(w) := { 1, self-bounding property} 2 { 2, L(ws )  1/4C ⌘ } 1 <latexit sha1_base64="w2KNB8ex1F9thBmvzfwkZfYLohI=">AAACb3icfVBdaxNBFJ1sq7bxox8++FAog0FoRdLdElofC/GhDxUrmLbQieHu5G4ydGZ2mblrDUt+gL/GV/tT+jP8B86mETQVL8xwOPdczr0nLbTyFMe3jWhp+cHDRyurzcdPnj5bW9/YPPN56ST2ZK5zd5GCR60s9kiRxovCIZhU43l61a3751/QeZXbTzQpsG9gZFWmJFCgBuutkx1hgMZpxq8HfpcLjTzZ63QHIkWCz/si/EEVt+NZ8fsgmYMWm9fpYKPREcNclgYtSQ3eXyZxQf0KHCmpcdoUpccC5BWM8DJACwZ9v5pdM+WvAjPkWe7Cs8Rn7J8TFRjvJyYNynpzv9iryX/2UrPgTNnbfqVsURJaeWeclZpTzuuo+FA5lKQnAYB0KuzO5RgcSAqBNpviHYbjHL4PRh8KdEC5e10JcCOj7DQcOxJvavQ/IXz9LQwo5JwspnofnO23k4P2wcdO66g7T3yFbbGXbIcl7JAdsWN2ynpMsm/sO/vBbho/oxfRdsTvpFFjPvOc/VXR7i/Qyr0X</latexit> 2 立 大 2 2 む n X 1 n i=1 立 1) L(wt )  1/4C ⌘ 2 <latexit sha1_base64="i0WPSQWTa/IGZW9g9tBbMHRtNiA=">AAACb3icfVBdaxNBFJ1sq7bxox8++FAog0FoRdLdElofC/GhDxUrmLbQieHu5G4ydGZ2mblrDUt+gL/GV/tT+jP8B86mETQVL8xwOPdczr0nLbTyFMe3jWhp+cHDRyurzcdPnj5bW9/YPPN56ST2ZK5zd5GCR60s9kiRxovCIZhU43l61a3751/QeZXbTzQpsG9gZFWmJFCgBuutkx1hgMZpxq8HtMuFRp7sdboDkSLB530R/qCK2/Gs+H2QzEGLzet0sNHoiGEuS4OWpAbvL5O4oH4FjpTUOG2K0mMB8gpGeBmgBYO+X82umfJXgRnyLHfhWeIz9s+JCoz3E5MGZb25X+zV5D97qVlwpuxtv1K2KAmtvDPOSs0p53VUfKgcStKTAEA6FXbncgwOJIVAm03xDsNxDt8How8FOqDcva4EuJFRdhqOHYk3NfqfEL7+FgYUck4WU70PzvbbyUH74GOnddSdJ77CtthLtsMSdsiO2DE7ZT0m2Tf2nf1gN42f0YtoO+J30qgxn3nO/qpo9xfSx70Y</latexit> : 2 L(ws )  1/4C ⌘ <latexit sha1_base64="YF4J7emCgB/vr4rN83sWlIWVB6g=">AAACT3icfZDPShxBEMZ7VhN1TOKfHHNpXASRsMyIqJeAYA65SAw4urCzSE1vzdrY3TN010iWYZ/Bq3ksjz5JbsHezQi6igUNP776mqr6slJJR1F0H7Tm5t+9X1hcCpc/fPy0srq2fuaKygpMRKEK283AoZIGE5KksFtaBJ0pPM+ujib982u0ThbmlEYl9jUMjcylAPJSQvwbdxer7agTTYu/hLiBNmvq5GIt2E0Hhag0GhIKnOvFUUn9GixJoXAcppXDEsQVDLHn0YBG16+n2475plcGPC+sf4b4VH36owbt3Ehn3qmBLt1sbyK+2sv0zGTKD/q1NGVFaMT/wXmlOBV8EgUfSIuC1MgDCCv97lxcggVBPrAwTL+jP87isR/0s0QLVNjtOgU71NKM/bHD9OuE3jLC70ejJ59zPJvqSzjb6cR7nb1fu+3DoybxRfaFbbAtFrN9dsh+sBOWMMEku2G37E9wF/wN/rUaayto4DN7Vq2lBwvLs4c=</latexit> 3 1 | g(hw, zi i)
  104. self-bounding property 0 <latexit sha1_base64="ju+VzU1qNrLTMVB0zM++F5jDHUQ=">AAADaniclVJdTxNBFJ3t+oH1C/DJEJOJ1YgUm11C0EeS8uCLERMLJEzdzE7vLhNmZzczd4W69M1f46v+Gf4DP4LZUqKWQvQmk5x77zk5d+5MXChpMQhOvYZ/6/adu3P3mvcfPHz0eH5hccfmpRHQE7nKzV7MLSipoYcSFewVBngWK9iND7t1f/crGCtz/RmHBfQznmqZSMHRlaIF7xkDpZaZ4jpVQFnG8SBO6FFUYTscrV7m3yLJzJjymtW06zQzFe2a/urf+dcM8+Z3jjNk7W7EYkBO0/9wOrnBg518WYvmW0EnGAe9CsIJaJFJbLuFrrNBLsoMNArFrd0PgwL7FTcohYJRk5UWCi4OeQr7Dmqege1X44cc0ZeuMqBJbtzRSMfVPxUVz6wdZrFj1nPa6V5dnNmLsylnTN71K6mLEkGLC+OkVBRzWv8SOpAGBKqhA1wY6Wan4oAbLtD9pWaTbYG7nIEPzuhjAYZjblYqxk2aST1yl03Zao1uIvLjS6JDbs/h9Favgp21TrjR2fi03trsTjY+R5bIc7JMQvKWbJL3ZJv0iPC+ez+8n96vxpm/6D/1ly6oDW+ieUL+Cv/FOTiGGkE=</latexit> `(hwt+1 , zi i) 

    `(hwt , zi i) + ` (hwt , zi i)hwt+1 data 課 1 G(w) := n <latexit sha1_base64="cxVTgSzQ8+j278159BuRivOEh6Q=">AAACoXicfVFdaxNBFJ2sXzV+pfroy2AQUikhK6WWQqFQQX0Qo5i2kInL3cnd7dCZ2WXmrjYu+WX+Eh991T/hbJqApuKFgcM553LvPZOWWnkaDL63omvXb9y8tXG7fefuvfsPOpsPj31ROYkjWejCnabgUSuLI1Kk8bR0CCbVeJKeHzX6yWd0XhX2I81KnBjIrcqUBApU0hm96gkDdJZm/MsW3z/ggvCCPM00cpE5kLEVvjJJrQ7i+Seb94QGmzfiqmt7hb4mSriFuJV0uoP+YFH8KoiXoMuWNUw2WztiWsjKoCWpwftxPChpUoMjJTXO26LyWII8hxzHAVow6Cf14v45fxqYKc8KF54lvmD/7KjBeD8zaXA2u/p1rSH/qaVmbTJle5Na2bIitPJycFZpTgVvwuVT5VCSngUA0qmwO5dnEFKk8AXttniJ4TiHb8OgdyU6oMI9qwW43Cg7D8fmYrtB/zPCxcoYUMg5Xk/1Kjh+3o93+7vvd7qHR8vEN9hj9oT1WMxesEP2mg3ZiEn2jf1gP9mvqBu9iYbRh0tr1Fr2PGJ/VTT+DYDr0Rw=</latexit> <latexit sha1_base64="GSu1euDo1zXqEtl+vixuKe17TVA=">AAAD+HichVLbbtNAEN3YXIq59fLIy4qIqiVtFJeo9AWpECR4AFEk0lbqhmi8WaerrtfW7roQXP8Lbwjxxt/wN6ydVErtEkaydHZmzpyZ8QSJ4Np0On8ajnvj5q3bS3e8u/fuP3i4vLJ6qONUUdansYjVcQCaCS5Z33Aj2HGiGESBYEfBWa+IH50zpXksP5lJwgYRjCUPOQVjXcOVxq93GyQCcxqE+MswMy0/3/TWiWB43m82cQsTAXJcC2xV2NtzQaKmjFZvSAJmAL+5QiUXi7gXn3cwId76i2on26QoZckSAgGVaEm71CsS7bOm+k9ioXfN7KXiAl6LhApokdT9X/X6NLgkZ88Kep518wUVhsvNTrtTGq4DfwaaaGYH9vd2ySimacSkoQK0PvE7iRlkoAynguUeSTVLgJ7BmJ1YKCFiepCVZ5XjJ9YzwmGs7CcNLr3zjAwirSdRYDOLRnU1VjivjQVRRdmEe4OMyyQ1TNKpcJgKbGJc3CweccWoERMLgCpue8f0FOzWjL1szyOvmR1OsfdW6EPCFJhYPc0IqHHEZW6HHZOtAi1KhK+XiRbZPfvVrdbB4U7b323vfuw293uzjS+hR+gx2kA+eo720Vt0gPqIOqvOnvPSeeV+c7+7P9yf01SnMeOsoSvm/v4LT3hDJQ==</latexit> L(wt+1 )  L(wt ) + hL(wt ), wt+1 2 Pn wt , zi i + C g(hwt , zi i)kwt+1 i=1 g(hw, zi i) wt i + C G(wt )kwt+1 2 2 = L(wt ) ⌘krL(wt )k + C ⌘ G(wt )krL(wt )k L(wt+1 )  L(wt ) + hL(wt ), wt+1 w i + C G(w )kw ⌘ t t t+1 2 2 Stage gradient factor G  L(wt )1 G(w ⌘krL(w )k + krL(w )k t 2⌘ t t )  1/4C 2 2 4 = L(wt ) ⌘krL(wt )k + C ⌘ G(wt )krL(wt )k 3⌘ 2 ⌘ 2 2 = L(w ) krL(w )k t t  L(wt ) ⌘krL(w )k + krL(w )k t t 4 4 3⌘ 2 Stage : = L(wt ) krL(wt )k 4 wt k 2 wt k 2 <latexit sha1_base64="GSu1euDo1zXqEtl+vixuKe17TVA=">AAAD+HichVLbbtNAEN3YXIq59fLIy4qIqiVtFJeo9AWpECR4AFEk0lbqhmi8WaerrtfW7roQXP8Lbwjxxt/wN6ydVErtEkaydHZmzpyZ8QSJ4Np0On8ajnvj5q3bS3e8u/fuP3i4vLJ6qONUUdansYjVcQCaCS5Z33Aj2HGiGESBYEfBWa+IH50zpXksP5lJwgYRjCUPOQVjXcOVxq93GyQCcxqE+MswMy0/3/TWiWB43m82cQsTAXJcC2xV2NtzQaKmjFZvSAJmAL+5QiUXi7gXn3cwId76i2on26QoZckSAgGVaEm71CsS7bOm+k9ioXfN7KXiAl6LhApokdT9X/X6NLgkZ88Kep518wUVhsvNTrtTGq4DfwaaaGYH9vd2ySimacSkoQK0PvE7iRlkoAynguUeSTVLgJ7BmJ1YKCFiepCVZ5XjJ9YzwmGs7CcNLr3zjAwirSdRYDOLRnU1VjivjQVRRdmEe4OMyyQ1TNKpcJgKbGJc3CweccWoERMLgCpue8f0FOzWjL1szyOvmR1OsfdW6EPCFJhYPc0IqHHEZW6HHZOtAi1KhK+XiRbZPfvVrdbB4U7b323vfuw293uzjS+hR+gx2kA+eo720Vt0gPqIOqvOnvPSeeV+c7+7P9yf01SnMeOsoSvm/v4LT3hDJQ==</latexit> 小 十 2 2 自 3 <latexit sha1_base64="Yhb1pcehKIiJ2QBW9IYKNuMU7EM=">AAACbXicfVBdaxNBFJ2sVWvqR1rxSZHRIFaRuFtC9bEQwb6ILZi20AnL3cnddOjM7DJzVw1Lnv01fW1/S3+Ff8HZNIKm4oULh3PP5dx7slIrT3F82YpurNy8dXv1Tnvt7r37DzrrGwe+qJzEoSx04Y4y8KiVxSEp0nhUOgSTaTzMTgfN/PArOq8K+4WmJY4MTKzKlQQKVNp59nFTGKCTLOffUnrFhUaevO0PUpEhgQiddrpxL54Xvw6SBeiyRe2l662+GBeyMmhJavD+OIlLGtXgSEmNs7aoPJYgT2GCxwFaMOhH9fyXGX8RmDHPCxfaEp+zf27UYLyfmiwom7v98qwh/znLzJIz5e9HtbJlRWjllXFeaU4Fb4LiY+VQkp4GANKpcDuXJ+BAUoiz3RYfMDzn8FMw+lyiAyrc61qAmxhlZ+HZiXjToP8J4ftvYUAh52Q51evgYKuXbPe29/vdncEi8VX2mD1nmyxh79gO22V7bMgk+8HO2Dm7aP2MHkVPoqdX0qi12HnI/qro5S9V2bxv</latexit> 3 1 | 2 wt k
  105. 1/t <latexit sha1_base64="B55qkJQFu/wxuKIQtEn5qJEykyM=">AAAClnicfVBdaxNBFJ2sXzV+NLUvBV8Gg5CKhl0p1RdLoYoiSiOYttCJy+zs3c2Qmdll5q5tWPY/+WsEn/SnOJumoKl4YeDcc8/lzjlJqaTDMPzRCa5dv3Hz1trt7p279+6v9zYeHLmisgLGolCFPUm4AyUNjFGigpPSAteJguNkdtDOj7+CdbIwn3FewkTz3MhMCo6einvvWWa5iJC5Ssf17FXYfKnxWdR8GDDNcZpk9VkTz7aZAsrOZAooVQr00PcZDi53rcynuB33+uEwXBS9CqIl6JNljeKNzg5LC1FpMCgUd+40Ckuc1NyiFAqaLqsclFzMeA6nHhquwU3qhemGPvZMSrPC+meQLtg/N2qunZvrxCtbJ2511pL/nCV65TJmLye1NGWFYMTF4axSFAvaJkpTaUGgmnvAhZX+71RMuU8Gfe7dLnsN3pyFj/7QYQmWY2Gf1IzbXEvTeLM5e9qi/wn5+aXQI59ztJrqVXD0fBjtDnc/7fT3D5aJr5GH5BEZkIi8IPvkHRmRMRHkG/lOfpJfwVawF7wJ3l5Ig85yZ5P8VcHoN/11zZQ=</latexit> 1 t t 1 X k=0 ✓

    ◆ 1 e L(wk )  O t 1 L(ws )  2 4C ⌘ <latexit sha1_base64="CIPUskdt35EwdEN016oSOaWhlSE=">AAACdnicfVBdaxNBFJ1s/ajxK62PggyGYi0SdkuofSzEBx8qVjBtoROXu5O76dCZ2WXmrjUs+x/8Nb7q3/Cf+NjZNIKm4oUZDueey73nZKVWnuL4Zydau3X7zt31e937Dx4+etzb2Dz2ReUkjmWhC3eagUetLI5JkcbT0iGYTONJdjFq+yef0XlV2I80L3FiYGZVriRQoNLezuG2MEDnWc4vU/9SaOQidyDrpKmHo1RkSPBpV4S/SXv9eBAvit8EyRL02bKO0o3OUEwLWRm0JDV4f5bEJU1qcKSkxqYrKo8lyAuY4VmAFgz6Sb0w1fCtwEx5XrjwLPEF++dEDcb7ucmCsjXgV3st+c9eZlY2U74/qZUtK0IrrxfnleZU8DYxPlUOJel5ACCdCrdzeQ4hIwq5drviDQZzDt+FRe9LdECF26kFuJlRtglmZ+JVi/4nhC+/hQGFnJPVVG+C491BsjfY+zDsH4yWia+zp+w522YJe80O2Ft2xMZMsq/sG/vOfnR+Rc+irejFtTTqLGeesL8qiq8AHqLA9g==</latexit> self-bounding 1 G(ws )  4C ⌘ <latexit sha1_base64="0z4Erz93vJLMBn0y95qSgdAdR+4=">AAACdHicfVDRahNBFJ2sVmusmtpHfRgMhSgSdiVUHwsp2JdiBdMWOmG5O7mbDp2ZXWbuqmHZX/BrfNX/8Ef63Nk0gqbihQuHc8/l3HuyUitPcfyrE925u3Hv/uaD7sOtR4+f9LafnviichInstCFO8vAo1YWJ6RI41npEEym8TS7HLfz08/ovCrsJ1qUODUwtypXEihQaW/wfiAM0EWW8y+pfyk0cpE7kHXS1KNxKjIkEKGbtNePh/Gy+G2QrECfreo43e6MxKyQlUFLUoP350lc0rQGR0pqbLqi8liCvIQ5ngdowaCf1suXGr4bmBnPCxfaEl+yf27UYLxfmCwo2/P9+qwl/znLzJoz5e+mtbJlRWjljXFeaU4Fb/PiM+VQkl4EANKpcDuXFxASopBqtysOMDzn8CgYfSjRARXuVS3AzY2yTXh2Ll636H9C+PpbGFDIOVlP9TY4eTNM9oZ7H0f9/fEq8U32jL1gA5awt2yfHbJjNmGSfWPf2Q/2s3MVPY/60e6NNOqsdnbYXxUNrwGaoMBN</latexit> 2 2 <latexit sha1_base64="TXIXLp1XEoQqZJ8+120LMpm7cgc=">AAACzXicfVHfa9RAEN7EXzVavSrogy+Lh9BaPZJSWh8LJ+iD0gpeW+iex2Rvcl262YTdSfXIneCTf6N/gf+Gm+sJNVUHFr795htm5pu01MpRHP8IwmvXb9y8tXI7unN39d79ztqDQ1dUVuJAFrqwxyk41MrggBRpPC4tQp5qPErP+k3+6BytU4X5SNMShzlMjMqUBPLUqPPt3brIgU7TjH8e1bSZzDcioZFfpmmDvxRIwMVMGEg1tLJi9mmLb/ZHIvWiRui/b1qKfxeOOt24Fy+CXwXJEnTZMg5Ga8G2GBeyytGQ1ODcSRKXNKzBkpIa55GoHJYgz2CCJx4ayNEN64VXc/7MM2OeFdY/Q3zBXq6oIXdumqde2Uzq2rmG/GsuzVudKXs1rJUpK0IjLxpnleZU8OYQfKwsStJTD0Ba5Wfn8hQsSPLniiLxGv1yFt/7RvslWqDCPq8F2EmuzNwvOxEvGvQ/IXz5LfTI+5y0Xb0KDrd6yU5v58N2d6+/dHyFPWFP2TpL2C7bY2/ZARswyX4Gq8Gj4HG4H1bhLPx6IQ2DZc1D9keE338BD9Hcww==</latexit> 大 小 小 十 十 4 大 L(wt+1 )  L(wt ) 3 1 | 2 2 ⌘krL(wt )k + C ⌘ G(wt )krL(wt )k 2
  106. 1 <latexit sha1_base64="XXmOklUwPUleckn8zhNpg+kIfVw=">AAAConicfVFNaxsxEJU3/UjcL6c99iLqFkoIZjeEtMdAciiF0oTWTiAyZlae3YhI2kWaTWo2/mf9I7322v6IaJ0tuE7pwMDTmzea0VNaauUpjn90orV79x88XN/oPnr85Omz3ubzkS8qJ3EoC1240xQ8amVxSIo0npYOwaQaT9KLg6Z+conOq8J+pVmJYwO5VZmSQIGa9EbEhUbvvTIBgM01cmGAztOMX01oe+mwJVxbb/J6SSWuly8pck6TXj8exIvgd0HSgj5r42iy2dkV00JWBi1JDd6fJXFJ4xocKalx3hWVxxLkBeR4FqAFg35cLwyY8zeBmfKscCEt8QW73FGD8X5m0qBstvartYb8Zy01K5Mpez+ulS0rQitvB2eV5lTwxl0+VQ4l6VkAIJ0Ku3N5Dg4khT/odsUhhsc5/BQGfS7RARVuqxbgcqPsPDw2F9sN+p8Qvv0RBhR8TlZdvQtGO4Nkb7B3vNvfP2gdX2cv2Sv2liXsHdtnH9gRGzLJvrOf7Bf7Hb2OPkbH0ZdbadRpe16wvyISN2M/0eo=</latexit> t . hwt , w⇤ i 

    kwt k . log t t 1 X ⌘ <latexit sha1_base64="l+krq2lSn/H3UCw2KgBOccURQCg=">AAACq3icfZFNb9QwEIa94assH93CkYvFCqmgdpWgqnCpVKkcuCCKYNuKeokm3klqxXYi2yksVn4eP4LfwBXuOOlWgi1iJEuPZt7RzLzOaimsi+Pvg+ja9Rs3b63dHt65e+/++mjjwZGtGsNxyitZmZMMLEqhceqEk3hSGwSVSTzOyoOufnyOxopKf3CLGmcKCi1ywcGFVDpKWYGUzStnaU+5Ae5ZAUoBQwet1y2zjUp9uRe3n7zbTtpik0nQhQxqBe4sy+nntNy65K+pF2nZMtNLnqajcTyJ+6BXIVnCmCzjMN0Y7IR1eKNQOy7B2tMkrt3Mg3GCS2yHrLFYAy+hwNOAGhTame+daOmTkJnTvDLhaUf77J8dHpS1C5UFZbevXa11yX/WMrUy2eUvZ17ounGo+cXgvJHUVbSzmc6FQe7kIgBwI8LulJ9BMNeFzxgO2SsMxxl8Ewa9rdGAq8wzz8AUSug2HFuwrY7+J4Qvl8JAwedk1dWrcPR8kuxOdt/tjPcPlo6vkUfkMdkkCXlB9slrckimhJNv5Af5SX5F29H76GPELqTRYNnzkPwVEf4GmrLWvQ==</latexit> ⌘ hwt 1 , w⇤ i + g(hwt 1 , zit 1 i) n 0 <latexit sha1_base64="+F6QNd7hhlFxIO/Fr3Q+hUWCTOY=">AAAC7nichVFda9RAFJ3Ej67xa6uPvgwuQq11SaS0fSy0D76IFdy20FnCzexNOnRmEmYm1TXkb/gmBZ/8S/4bJ7tb1N2iFwYO5565H+dmlRTWxfHPILx1+87dtd696P6Dh48e99efHNuyNhxHvJSlOc3AohQaR044iaeVQVCZxJPs4qDLn1yisaLUH920wrGCQotccHCeSvvfmQRdSKRMgTvPcvopdVu/8SYzs3TECi9ZkTbuddLeIKevKMsN8IYVoBQwdNA2ui02/lfiS9qIOdVel3qZ9gfxMJ4FXQXJAgzIIo7S9WCbTUpeK9SOS7D2LIkrN27AOMElthGrLVbAL6DAMw81KLTjZmZmS194ZkLz0vinHZ2xf/5oQFk7VZlXdkPb5VxH3pjL1FJnl++NG6Gr2qHm88Z5LakraXcpOhEGuZNTD4Ab4Wen/By8q87fM4rYIfrlDL7zjd5XaMCVZrNhYAoldOuXLdhWh/4lhM/XQo+8z8myq6vg+M0w2RnufNge7B8sHO+RZ+Q52SAJ2SX75C05IiPCg14wDHaDvbAKv4bfwqu5NAwWf56SvyL88QtMROyu</latexit> hwt , w⇤ i <latexit sha1_base64="lcEyyIKdACMSutjHiGMPtGiGuss=">AAACZnicfZDfShtBFMYnW21ttBpbSqG9GQylViTsFrGlUBDsRW9EBaOCG8LZyclmcP4sM2dLwxro0/S2fZ2+QR+jk7iCjdIDAz++8x3OnC8rlPQUx78b0YOFxYePlh43l1eerK611p+eels6gV1hlXXnGXhU0mCXJCk8LxyCzhSeZZf70/7ZV3ReWnNC4wJ7GnIjh1IABanfeplvpmJg6S3/+IlfpajUm1q46rfacSeeFb8LSQ1tVtdRf72xkw6sKDUaEgq8v0jignoVOJJC4aSZlh4LEJeQ40VAAxp9r5odMeGvgzLgQ+vCM8Rn6u2JCrT3Y50FpwYa+fneVLy3l+m5zTT80KukKUpCI64XD0vFyfJpQnwgHQpS4wAgnAx/52IEDgSFHJvN9DOG4xwehEWHBTog67aqFFyupZmEY/N0e0r/M8K3G2OgkHMyn+pdOH3XSXY7u8c77b39OvEl9optsE2WsPdsj31hR6zLBPvOfrCf7FfjT7QaPY9eXFujRj3zjP1TEf8LTaK5rA==</latexit> g(·) := |` (·)| per-sample loss gradient ⌘✏ t g(hwk , zik i) & n n k=0 ··· t−1 <latexit sha1_base64="y0AsmGk4CwZls4eiFewBo6cAAgI=">AAACcXicfVDbahRBEO0db3G9bfRJfGmyBERlmZEQfQzEB1/ECG4SSC9LTW/NpElfhu4acRn2C/waX/VL/A5/wNrNCroRC4o+nHOK6jplY02iPP/Ry65dv3Hz1tbt/p279+4/GGw/PE6hjRrHOtgQT0tIaI3HMRmyeNpEBFdaPCkvDpf6ySeMyQT/keYNThzU3lRGAzE1Heyqmlh1UlURdKdqcA4UEneTjA1+0fkFTQfDfJSvSl4FxRoMxbqOptu9PTULunXoSVtI6azIG5p0EMloi4u+ahM2oC+gxjOGHhymSbe6ZyF3mZnJKkRuT3LF/jnRgUtp7kp2OqDztKktyX9qpdvYTNXrSWd80xJ6fbm4aq2kIJdhyZmJqMnOGYCOhv8u9TlwUMSR9vvqDfJxEd/xovcNRqAQn3UKYu0MB8everFE/zPC599GRpxzsZnqVXD8clTsj/Y/7A0PDteJb4knYkc8FYV4JQ7EW3EkxkKLL+Kr+Ca+935mjzOZ7Vxas9565pH4q7LnvwAt6cC9</latexit> ≳ε ↔︎ 立 ( 示 6 η 3 1 | Perceptron argument ε )
  107. w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit> zi 8 <latexit sha1_base64="7LY/jY56Y76zvHv7R8rq4Ttd7PQ=">AAACVnicfZDdSsNAEIU38ae1/rV66U2wCCJSEhH1slAvvBErWBVMKZPtpC7ubsLuRqyhr+GtPpa+jLipFbSKAwsfZ84yMydKOdPG998cd2Z2br5UXqgsLi2vrFZra5c6yRTFDk14oq4j0MiZxI5hhuN1qhBExPEqumsV/at7VJol8sIMU+wKGEgWMwrGSmEowNxGcf446rFete43/HF5vyGYQJ1Mqt2rOfthP6GZQGkoB61vAj813RyUYZTjqBJmGlOgdzDAG4sSBOpuPl565G1Zpe/FibJPGm+sfv+Rg9B6KCLrLJbU071C/LMXianJJj7q5kymmUFJPwfHGfdM4hWJeH2mkBo+tABUMbu7R29BATU2t0olPEZ7nMJTO+gsRQUmUTt5CGogmBzZYwfhbkH/GeHhy2jJ5hxMp/obLvcawUHj4Hy/3mxNEi+TDbJJtklADkmTnJA26RBKUvJEnsmL8+q8u3Nu6dPqOpM/6+RHudUP8/G2cg==</latexit> 3 1 hw,

    zi i > 0 <latexit sha1_base64="1cEtjBRSN+vWN3nw8IqXksvbm7I=">AAACdXicfZDfahNBFMYna9WY+iepl1IYjIpKibsSYq+k0F54U1qhSQvZEM5Ozm6HzswuM7PVdNln6NP0Vp+jT9JbZ9MEalJ6YODH953hnPNFmeDG+v51zXu09vjJ0/qzxvrzFy9fNVsbA5PmmmGfpSLVJxEYFFxh33Ir8CTTCDISeByd7Vb+8Tlqw1N1ZKcZjiQkisecgXXSuPkpFKASgaEEexrFxa9ya4EX5ZiHeubS79QfN9t+x58VXYVgDm0yr8Nxq9YNJynLJSrLBBgzDPzMjgrQljOBZSPMDWbAziDBoUMFEs2omN1U0vdOmdA41e4pS2fq3R8FSGOmMnKd1b5m2avEe71ILk228fao4CrLLSp2OzjOBbUprQKjE66RWTF1AExztztlp6CBWRdroxHuoTtO474bdJChBpvqz0UIOpFcle7YJNyq6KFG+L1odORyDpZTXYXB107Q6/R+dts7u/PE6+QNeUs+koB8IzvkBzkkfcLIJbkif8jf2o236b3zPty2erX5n9fkv/K+/AMvsMGB</latexit>
  108. w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit> zj 9 <latexit sha1_base64="z8pIRFgTl90+mGRs6e8QA6hLT4U=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GCVcGUMtlO6trdTdjdiDX0NbzVx9KXETe1glZxYOHjzFlm5kQpZ9r4/pvjTk3PzM6V5ssLi0vLK5XVtUudZIpiiyY8UdcRaORMYssww/E6VQgi4ngV9RtF/+oelWaJvDCDFNsCepLFjIKxUhgKMLdRnD8OO3edStWv+aPyfkMwhioZV7Oz6uyH3YRmAqWhHLS+CfzUtHNQhlGOw3KYaUyB9qGHNxYlCNTtfLT00NuySteLE2WfNN5I/f4jB6H1QETWWSypJ3uF+GcvEhOTTXzUzplMM4OSfg6OM+6ZxCsS8bpMITV8YAGoYnZ3j96CAmpsbuVyeIz2OIWndtBZigpMonbyEFRPMDm0x/bC3YL+M8LDl9GSzTmYTPU3XO7VgoPawfl+td4YJ14iG2STbJOAHJI6OSFN0iKUpOSJPJMX59V5d2fcuU+r64z/rJMf5VY+APXYtnM=</latexit> 3 1 hw,

    zi i > 0 <latexit sha1_base64="1cEtjBRSN+vWN3nw8IqXksvbm7I=">AAACdXicfZDfahNBFMYna9WY+iepl1IYjIpKibsSYq+k0F54U1qhSQvZEM5Ozm6HzswuM7PVdNln6NP0Vp+jT9JbZ9MEalJ6YODH953hnPNFmeDG+v51zXu09vjJ0/qzxvrzFy9fNVsbA5PmmmGfpSLVJxEYFFxh33Ir8CTTCDISeByd7Vb+8Tlqw1N1ZKcZjiQkisecgXXSuPkpFKASgaEEexrFxa9ya4EX5ZiHeubS79QfN9t+x58VXYVgDm0yr8Nxq9YNJynLJSrLBBgzDPzMjgrQljOBZSPMDWbAziDBoUMFEs2omN1U0vdOmdA41e4pS2fq3R8FSGOmMnKd1b5m2avEe71ILk228fao4CrLLSp2OzjOBbUprQKjE66RWTF1AExztztlp6CBWRdroxHuoTtO474bdJChBpvqz0UIOpFcle7YJNyq6KFG+L1odORyDpZTXYXB107Q6/R+dts7u/PE6+QNeUs+koB8IzvkBzkkfcLIJbkif8jf2o236b3zPty2erX5n9fkv/K+/AMvsMGB</latexit>
  109. w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit> zj <latexit sha1_base64="z8pIRFgTl90+mGRs6e8QA6hLT4U=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GCVcGUMtlO6trdTdjdiDX0NbzVx9KXETe1glZxYOHjzFlm5kQpZ9r4/pvjTk3PzM6V5ssLi0vLK5XVtUudZIpiiyY8UdcRaORMYssww/E6VQgi4ngV9RtF/+oelWaJvDCDFNsCepLFjIKxUhgKMLdRnD8OO3edStWv+aPyfkMwhioZV7Oz6uyH3YRmAqWhHLS+CfzUtHNQhlGOw3KYaUyB9qGHNxYlCNTtfLT00NuySteLE2WfNN5I/f4jB6H1QETWWSypJ3uF+GcvEhOTTXzUzplMM4OSfg6OM+6ZxCsS8bpMITV8YAGoYnZ3j96CAmpsbuVyeIz2OIWndtBZigpMonbyEFRPMDm0x/bC3YL+M8LDl9GSzTmYTPU3XO7VgoPawfl+td4YJ14iG2STbJOAHJI6OSFN0iKUpOSJPJMX59V5d2fcuU+r64z/rJMf5VY+APXYtnM=</latexit> hw, zj i >

    0 0 <latexit sha1_base64="GSgrp7N7ZewU7vBLaf0+Kfpt6Bo=">AAACdXicfZDfahNBFMYnq9UYq031UgpDo6IS0l0prVcSSC96I41g2kI3hLOTs5tpZ2aXmVltXPYZfBpv9Tn6JL3tbP5Am4oHBn583xnOOV+UCW6s71/VvAcP1x49rj9pPF1/9nyjufni2KS5ZjhgqUj1aQQGBVc4sNwKPM00gowEnkQXvco/+Y7a8FR9s9MMhxISxWPOwDpp1HwfClCJwFCCnURx8aNsL/FnOToP9cyln6k/arb8jj8reh+CBbTIovqjzdpuOE5ZLlFZJsCYs8DP7LAAbTkTWDbC3GAG7AISPHOoQKIZFrObSvrGKWMap9o9ZelMvf2jAGnMVEaus9rXrHqV+E8vkiuTbfxpWHCV5RYVmw+Oc0FtSqvA6JhrZFZMHQDT3O1O2QQ0MOtibTTCA3THafziBh1lqMGm+kMRgk4kV6U7NgnbFf2vES6XjY5czsFqqvfh+GMn2Ovsfd1tdXuLxOvkFdkm70hA9kmXHJI+GRBGfpHf5A/5W7v2trzX3tt5q1db/HlJ7pS3cwMxosGC</latexit> 4 1 hw, zi i > 0 <latexit sha1_base64="1cEtjBRSN+vWN3nw8IqXksvbm7I=">AAACdXicfZDfahNBFMYna9WY+iepl1IYjIpKibsSYq+k0F54U1qhSQvZEM5Ozm6HzswuM7PVdNln6NP0Vp+jT9JbZ9MEalJ6YODH953hnPNFmeDG+v51zXu09vjJ0/qzxvrzFy9fNVsbA5PmmmGfpSLVJxEYFFxh33Ir8CTTCDISeByd7Vb+8Tlqw1N1ZKcZjiQkisecgXXSuPkpFKASgaEEexrFxa9ya4EX5ZiHeubS79QfN9t+x58VXYVgDm0yr8Nxq9YNJynLJSrLBBgzDPzMjgrQljOBZSPMDWbAziDBoUMFEs2omN1U0vdOmdA41e4pS2fq3R8FSGOmMnKd1b5m2avEe71ILk228fao4CrLLSp2OzjOBbUprQKjE66RWTF1AExztztlp6CBWRdroxHuoTtO474bdJChBpvqz0UIOpFcle7YJNyq6KFG+L1odORyDpZTXYXB107Q6/R+dts7u/PE6+QNeUs+koB8IzvkBzkkfcLIJbkif8jf2o236b3zPty2erX5n9fkv/K+/AMvsMGB</latexit>
  110. wt <latexit sha1_base64="o8oLpubqJ4g9xtRuRm3MKOq0r4Q=">AAACVnicfVBdaxNBFJ3dWlvjV1offRkMgogsu2nIJm+F+uCLWMG0hW4Idyd306Ezs8vMXTUs+Rt9rT9L/4w4m6agRb1w4XDuuZx7T14p6SiOfwTh1r3t+zu7DzoPHz1+8rS7t3/iytoKnIhSlfYsB4dKGpyQJIVnlUXQucLT/PKonZ9+RutkaT7RssKphoWRhRRAnsoyDXSRF82X1Yxm3V4cxQfpaBDzOOqP01Ey9mCY9kfpmCdRvK4e29TxbC8YZPNS1BoNCQXOnSdxRdMGLEmhcNXJaocViEtY4LmHBjS6abM+esVfembOi9L6NsTX7O8bDWjnljr3yvZId3fWkn+d5fqOMxWjaSNNVRMacWNc1IpTydtE+FxaFKSWHoCw0t/OxQVYEORz63Syt+ifs/jeG32o0AKV9nWTgV1oaVb+2UX2pkX/E8LXW6FHPufbMPm/wUk/SobR8OOgd3i0SXyXPWcv2CuWsJQdsnfsmE2YYBW7YtfsW/A9+Bluhzs30jDY7Dxjf1TY/QW0abbb</latexit> 大 4 ⾒ 4 1 w⇤ <latexit

    sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit>
  111. Stage : ⟨wt, w*⟩ ≳ 1 ⋅ ε 大 1

    5 ⾒ 4 1 w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit>
  112. Stage : ⟨wt, w*⟩ ≳ 2 ⋅ ε 大 1

    6 ⾒ 4 1 w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit>
  113. Stage : ⟨wt, w*⟩ ≳ τ ⋅ ε 大 1

    7 ⾒ 4 1 w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit>
  114. Stage : <latexit sha1_base64="ma7DEH8yS3YCblisYhSz4SzZvxg=">AAACn3icfVHbbhMxEHW2XEq4pfDIiyFCaoFGuyVqeaxUJHjgEgRpi+oQzTqzqVXbu7JngWi7H8an8MQr/AXeNEglRYxk6ejMGZ2Z47TQylMcf29FK5cuX7m6eq19/cbNW7c7a3f2fV46iUOZ69wdpuBRK4tDUqTxsHAIJtV4kJ7sNf2Dz+i8yu0HmhU4MjC1KlMSKFDjzvtX68IAHacZ/zKu6HFSb3ChkZ+naYNvCsocyOqpQIK66tdcnAoLqYYlpTj9tDXudONePC9+ESQL0GWLGozXWn0xyWVp0JLU4P1REhc0qsCRkhrrtig9FiBPYIpHAVow6EfV/PqaPwzMhGe5C88Sn7PnJyow3s9MGpTNon6515D/7KVmyZmyZ6NK2aIktPLMOCs1p5w30fKJcihJzwIA6VTYnctjCLFR+IB2WzzHcJzD18HobYEOKHePKgFuapStw7FT8aRB/xPC1z/CgELOyXKqF8H+Vi/Z7m2/63d39xaJr7J77AFbZwnbYbvsJRuwIZPsG/vBfrJf0f3oRfQmGpxJo9Zi5i77q6KPvwE4hM8j</latexit> L(wt+1 )  L(wt ) 大

    2 8 ⾒ 4 1 w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit> 3⌘ 2 4 krL(wt )k
  115. Stage : <latexit sha1_base64="ma7DEH8yS3YCblisYhSz4SzZvxg=">AAACn3icfVHbbhMxEHW2XEq4pfDIiyFCaoFGuyVqeaxUJHjgEgRpi+oQzTqzqVXbu7JngWi7H8an8MQr/AXeNEglRYxk6ejMGZ2Z47TQylMcf29FK5cuX7m6eq19/cbNW7c7a3f2fV46iUOZ69wdpuBRK4tDUqTxsHAIJtV4kJ7sNf2Dz+i8yu0HmhU4MjC1KlMSKFDjzvtX68IAHacZ/zKu6HFSb3ChkZ+naYNvCsocyOqpQIK66tdcnAoLqYYlpTj9tDXudONePC9+ESQL0GWLGozXWn0xyWVp0JLU4P1REhc0qsCRkhrrtig9FiBPYIpHAVow6EfV/PqaPwzMhGe5C88Sn7PnJyow3s9MGpTNon6515D/7KVmyZmyZ6NK2aIktPLMOCs1p5w30fKJcihJzwIA6VTYnctjCLFR+IB2WzzHcJzD18HobYEOKHePKgFuapStw7FT8aRB/xPC1z/CgELOyXKqF8H+Vi/Z7m2/63d39xaJr7J77AFbZwnbYbvsJRuwIZPsG/vBfrJf0f3oRfQmGpxJo9Zi5i77q6KPvwE4hM8j</latexit> L(wt+1 )  L(wt ) 大

    2 9 ⾒ 4 1 w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit> 3⌘ 2 4 krL(wt )k
  116. Stage : <latexit sha1_base64="ma7DEH8yS3YCblisYhSz4SzZvxg=">AAACn3icfVHbbhMxEHW2XEq4pfDIiyFCaoFGuyVqeaxUJHjgEgRpi+oQzTqzqVXbu7JngWi7H8an8MQr/AXeNEglRYxk6ejMGZ2Z47TQylMcf29FK5cuX7m6eq19/cbNW7c7a3f2fV46iUOZ69wdpuBRK4tDUqTxsHAIJtV4kJ7sNf2Dz+i8yu0HmhU4MjC1KlMSKFDjzvtX68IAHacZ/zKu6HFSb3ChkZ+naYNvCsocyOqpQIK66tdcnAoLqYYlpTj9tDXudONePC9+ESQL0GWLGozXWn0xyWVp0JLU4P1REhc0qsCRkhrrtig9FiBPYIpHAVow6EfV/PqaPwzMhGe5C88Sn7PnJyow3s9MGpTNon6515D/7KVmyZmyZ6NK2aIktPLMOCs1p5w30fKJcihJzwIA6VTYnctjCLFR+IB2WzzHcJzD18HobYEOKHePKgFuapStw7FT8aRB/xPC1z/CgELOyXKqF8H+Vi/Z7m2/63d39xaJr7J77AFbZwnbYbvsJRuwIZPsG/vBfrJf0f3oRfQmGpxJo9Zi5i77q6KPvwE4hM8j</latexit> L(wt+1 )  L(wt ) 大

    2 0 ⾒ 5 1 w⇤ <latexit sha1_base64="sSazVUx+ioX3OkVvK3agBt30NXA=">AAACVnicfZDdSsNAEIU38a/Wv6qX3gSLICIlEVEvC/XCG7GC1YIpZbKd1MXdTdjdqCX0NbzVx9KXETe1grbiwMLHmbPMzIlSzrTx/XfHnZmdm18oLZaXlldW1yrrG9c6yRTFFk14otoRaORMYssww7GdKgQRcbyJ7htF/+YBlWaJvDKDFDsC+pLFjIKxUhgKMHdRnD8Ou3vdStWv+aPypiEYQ5WMq9lddw7DXkIzgdJQDlrfBn5qOjkowyjHYTnMNKZA76GPtxYlCNSdfLT00NuxSs+LE2WfNN5I/fkjB6H1QETWWSypJ3uF+GcvEhOTTXzSyZlMM4OSfg2OM+6ZxCsS8XpMITV8YAGoYnZ3j96BAmpsbuVyeIr2OIXndtBFigpMovbyEFRfMDm0x/bD/YL+M8LTt9GSzTmYTHUarg9qwVHt6PKwWm+MEy+RLbJNdklAjkmdnJEmaRFKUvJMXsir8+Z8uHPuwpfVdcZ/Nsmvciufdlq2MA==</latexit> 3⌘ 2 4 krL(wt )k
  117. : む ( : wt+1 = wt む ) <latexit

    sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> [Wu+ ⌘rL(wt ) ] [Bao+ ] η L(wt) ≤ ε 十 一 5 4 2 2 0 0 2 2 1 Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses. 5 1 📄 Self-bounding property
  118. : む ( : wt+1 = wt む ) <latexit

    sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> [Wu+ ] ⌘rL(wt ) [Bao+ ] = perceptron argument 一 5 4 2 2 0 0 2 2 2 Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses. 5 1 📄 大 Self-bounding property
  119. む ⾒ む perceptron argument む Edge of stability edge

    of stability EoS ; 見 見 見 非 大 3 む And others? 5 1 Open questions
  120. 白 面 大 2 大 一 十 大 6 5

    非 高 大 1 む : NN cf. む ;
  121. む w(0) ŵ ・ 力 力 力 7 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit>

    力 ⚙️ <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> 5 1 👑 👑 む ⚙️
  122. む む ŵ 立 ・ 用 用 力 方 8

    <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> 立 w(0) 力 ⚙️ <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> 5 1 👑 む