x' + D y y' = D (x + y) (x' + y') D x x' * D y y' = D (x * y) (x' * y + x * y') negate (D x x') = D (negate x) (negate x') abs (D x x') = D (abs x) (x' * (signum x)) signum (D x x') = D (signum x) 0 fromInteger n = D (fromInteger n) 0 instance Fractional a => Fractional (D a) where recip (D x x') = D (recip x) (-1 * x' * (recip (x * x))) fromRational x = D (fromRational x) 0 ϥΠϓχοπϧʔϧ {f(x)g(x)}′ = f′(x)g(x) + f(x)g′(x)
D pi 0 exp (D x x') = D (exp x) (x' * exp x) log (D x x') = D (log x) (x' / x) sin (D x x') = D (sin x) (x' * cos x) cos (D x x') = D (cos x) (- x' * sin x) asin (D x x') = D (asin x) (x' / (sqrt(1 - x ** 2))) acos (D x x') = D (acos x) (- x' / (sqrt(1 - x ** 2))) atan (D x x') = D (atan x) (x' / (1 + x ** 2)) sinh (D x x') = D (sinh x) (x' * cosh x) cosh (D x x') = D (cosh x) (x' * sinh x) asinh (D x x') = D (asinh x) (x' / (sqrt(1 + x ** 2))) acosh (D x x') = D (acosh x) (x' / (sqrt(x ** 2 - 1))) atanh (D x x') = D (atanh x) (x' / (1 - x ** 2))
x = D x 0 infinitesimal :: Num a => D a infinitesimal = D 0 1 diffD :: Num a => (D a -> D a) -> a -> a diffD f x = tangent $ f (lift x + infinitesimal) diffͷ࣮
Num a => Num (AD s a) where ... instance Fractional a => Fractional (AD s a) where ... instance Floating a => Floating (AD s a) where ... ༓ྶܕ > (1 :: AD Bool Int) + (2 :: AD Bool Int) AD {unAD = 3} > (1 :: AD Bool Int) + (2 :: AD Char Int) error: • Couldn't match type ‘Char’ with ‘Bool’
a) liftAD = AD . lift diffAD :: Num a => (forall s. AD s (D a) -> AD s (D a)) -> (a -> a) diffAD f = diffD (unAD . f . AD) ଘࡏܕ • diffAD ͷୈҰҾଘࡏܕʹͳ͍ͬͯΔ • s ͕۩ମతʹͲΜͳܕ͔֎ଆ͔Β͔Βͳ͍
+ y)^2) 1) 1 error: • Couldn't match type ‘s1’ with ’s’ > diffAD (\y -> diffAD (\x -> (x + liftAD y)^3) (liftAD 1)) 1 12 ˢ͜͜ͰΤϥʔʹͳΔ diffAD Λ࣮ߦͯ͠ΈΔ ؒҧͬͨํʹܕ͕͔ͭͳ͍ʂ AD s (D a) AD s1 (D a)
Reverse (unsafePerformIO (modifyTape (Proxy :: Proxy s) (bin i j di dj))) $! f b c partials (Reverse k _) = map (sensitivities !) [0..vs] where Head n t = unsafePerformIO $ readIORef (getTape (reflect (Proxy :: Proxy s))) ... ΈཱͯΔ࣌ʗ࣮ߦ࣌ʹ෭࡞༻͕ൃੜ͢Δ
2018. ʢܧଓ͠ελΠϧΛ͏͜ͱͰٯ͖ͷܭࢉΛ࣮ݱʣ • Fei Wang, et al; “Demystifying Differentiable Programming: Shift/Reset the Penultimate Backpropagator”, 2018. • Alois Brunel, et al; “Backpropagation in the Simply Typed Lambda- calculus with Linear Negation”, 2019. ʢઢܗܕΛ͏͜ͱͰಋ͕ؔʹධՁ͞Εͳ͍͜ͱΛอূ͢Δʣ • Robin Cockett, et al; “Reverse derivative categories”, 2019.