Let’s create a programming language! [RUG::B edition]

Transcript

1. Let’s create a
   programming

   language!
   DENIS DEFREYNE RUG
   :
   :B JUNE 2, 2016
   

   View Slide

2. 2
   dynamic vs. static
   

   View Slide

3. 3
   dynamic vs. static
   

   View Slide

4. 4
   compiler
   interpreter vs.
   

   View Slide

5. 5
   compiler
   interpreter vs.
   

   View Slide

6. but why?
   6
   

   View Slide

7. incremental
   7
   

   View Slide

8. Language 1
   8
   Numbers
   

   View Slide

9. %
   9
   cat stuff.cke
   4173
   5
   27
   2016
   

   View Slide

10. %
    10
    cat stuff.cke | ./clarke
    4173
    5
    27
    2016
    

    View Slide

11. 11
    module Grammar
    extend DParse::DSL
    DIGIT = char_in('0'..'9')
    PROGRAM = DIGIT
    end
    

    View Slide

12. 12
    res = Grammar::PROGRAM.apply($stdin.read)
    

    case res
    when DParse::Success
    handle_success(res)
    when DParse::Failure
    handle_failure(res)
    end
    

    View Slide

13. 13
    def handle_success(success)
    p success.data
    end
    

    View Slide

14. 14
    DIGIT = char_in('0'..'9')
    

    

    

    

    

    

    

    

    View Slide

15. 15
    DIGIT = …
    NUMBER = repeat1(DIGIT)
    

    

    

    

    

    View Slide

16. 16
    DIGIT = …
    NUMBER = repeat1(DIGIT)
    .capture
    

    

    

    

    

    View Slide

17. 17
    DIGIT = …
    NUMBER = repeat1(DIGIT)
    .capture
    .map { |d| d.to_i }
    

    

    

    

    

    View Slide

18. 18
    DIGIT = …
    NUMBER = …
    EXPRESSION = NUMBER
    

    

    

    

    

    

    View Slide

19. 19
    DIGIT = …
    NUMBER = …
    EXPRESSION = …
    EXPRESSIONS =
    intersperse(
    EXPRESSION,
    char("\n").ignore,
    ).compact
    

    View Slide

20. 20
    DIGIT = …
    NUMBER = …

    EXPRESSION = …
    EXPRESSIONS = …
    PROGRAM =
    seq(EXPRESSIONS, eof)
    .first

    

    

    View Slide

21. %
    21
    cat stuff.cke
    4173
    5
    27
    2016
    

    View Slide

22. %
    22
    cat stuff.cke | ./clarke
    [4173, 5, 27, 2016]
    

    View Slide

23. 23
    def handle_success(success)
    p success.data
    end
    

    View Slide

24. 24
    def handle_success(success)
    success.data.each do |expr|
    p eval_expr(expr)
    end
    end
    

    View Slide

25. 25
    def eval_expr(expr)
    expr
    end
    

    View Slide

26. %
    26
    cat stuff.cke | ./clarke
    4173
    5
    27
    2016
    

    View Slide

27. %
    27
    echo "aa" | ./clarke
    expected digit at line 1, column 1
    aa
    ↑
    

    View Slide

28. Language 2
    28
    Functions
    

    View Slide

29. %
    29
    cat stuff.cke
    4173
    add(2, 5)
    

    View Slide

30. %
    30
    cat stuff.cke | ./clarke
    4173
    7
    

    View Slide

31. 31
    LETTER =
    char_in('a'..'z')
    FUNCTION_NAME =
    repeat1(LETTER).capture
    

    View Slide

32. 32
    FUNCTION_CALL =
    seq(
    FUNCTION_NAME,
    char('('),
    intersperse(
    lazy { EXPRESSION },
    char(','),
    ),
    char(')'),
    )
    

    View Slide

33. 33
    FUNCTION_CALL =
    seq(
    FUNCTION_NAME,
    char('('),
    intersperse(
    lazy { EXPRESSION },
    char(',').ignore,
    ).compact,
    char(')'),
    )
    

    View Slide

34. 34
    FUNCTION_CALL =
    seq(
    FUNCTION_NAME,
    char('(').ignore,
    intersperse(
    lazy { EXPRESSION },
    char(',').ignore,
    ).compact,
    char(')').ignore,
    ).compact
    

    View Slide

35. 35
    EXPRESSION =
    NUMBER
    

    View Slide

36. 36
    EXPRESSION =
    alt(NUMBER, FUNCTION_CALL)
    

    View Slide

37. %
    37
    cat stuff.cke
    4173
    add(2, 5)
    

    View Slide

38. %
    38
    cat stuff.cke | ./clarke
    4173
    ["add", [2, 5]]
    

    View Slide

39. 39
    def eval_expr(expr)
    expr
    end
    

    View Slide

40. 40
    def eval_expr(expr)
    case expr
    when Integer
    expr
    when Array
    eval_function_call(expr)
    end
    end
    

    View Slide

41. 41
    def eval_function_call(expr)
    case expr[0]
    when 'add'
    expr[1]
    .map { |e| eval_expr(e) }
    .reduce(:+)
    when 'mul'
    …
    end
    end
    

    View Slide

42. %
    42
    cat stuff.cke
    4173
    add(2, 5)
    

    View Slide

43. %
    43
    cat stuff.cke | ./clarke
    4173
    7
    

    View Slide

44. Ugh, we forgot about whitespace.
    44
    

    View Slide

45. Cleanup 1
    45
    Abstract syntax
    

    View Slide

46. 46
    class IntNode < Struct.new(:value)
    end
    class FunCallNode < Struct.new(:name, :arguments)
    end
    

    View Slide

47. 47
    DIGIT = …
    NUMBER = repeat1(DIGIT)
    .capture
    .map { |d| d.to_i }
    

    

    

    

    

    View Slide

48. 48
    DIGIT = …
    NUMBER = repeat1(DIGIT)
    .capture
    .map { |d| IntNode.new(d.to_i) }
    

    

    

    

    

    View Slide

49. 49
    FUNCTION_CALL =
    seq(
    FUNCTION_NAME,
    char('(').ignore,
    intersperse(…),
    char(')').ignore,
    ).compact
    

    View Slide

50. 50
    FUNCTION_CALL =
    seq(
    FUNCTION_NAME,
    char('(').ignore,
    intersperse(…),
    char(')').ignore,
    ).compact.map do |data|
    FunCallNode.new(data[0], data[1])
    end
    

    View Slide

51. 51
    def eval_expr(expr)
    case expr
    when Integer
    expr
    when Array
    eval_function_call(expr)
    end
    end
    

    View Slide

52. 52
    def eval_expr(expr)
    case expr
    when IntNode
    expr.value
    when FunCallNode
    eval_function_call(expr)
    end
    end
    

    View Slide

53. 53
    def eval_function_call(expr)
    case expr[0]
    when 'add'
    expr[1]
    .map { |e| eval_expr(e) }
    .reduce(:+)
    when 'mul'
    …
    end
    end
    

    View Slide

54. 54
    def eval_function_call(expr)
    case expr.name
    when 'add'
    expr.args
    .map { |e| eval_expr(e) }
    .reduce(:+)
    when 'mul'
    …
    end
    end
    

    View Slide

55. Cleanup 2
    55
    Environment
    

    View Slide

56. 56
    class FunDef < Struct.new(:args, :body)
    end
    

    View Slide

57. 57
    $env = {}
    $env['add'] =
    FunDef.new(
    ['a', 'b'],
    ->(a, b) { a + b },
    )
    

    View Slide

58. 58
    def eval_function_call(expr)
    case expr.name
    when 'add'
    expr.args
    .map { |e| eval_expr(e) }
    .reduce(:+)
    when 'mul'
    …
    end
    end
    

    View Slide

59. 59
    def eval_function_call(expr)
    fun = $env[expr.name]
    values = expr.args.map { |e| eval_expr(e) }
    fun.body.call(*values)
    end
    

    View Slide

60. … with error handling, of course.
    60
    

    View Slide

61. Language 3
    61
    Variables
    

    View Slide

62. %
    62
    cat stuff.cke
    4173
    add(2,amount)
    

    View Slide

63. %
    63
    cat stuff.cke | ./clarke
    4173
    7
    

    View Slide

64. 64
    class VarNode < Struct.new(:name)
    end
    

    View Slide

65. 65
    VARIABLE_NAME =
    repeat1(LETTER).capture
    

    View Slide

66. 66
    VARIABLE_REFERENCE =
    VARIABLE_NAME
    .map { |d| VarNode.new(d) }
    

    View Slide

67. 67
    EXPRESSION =
    alt(

    NUMBER,

    FUNCTION_CALL,

    )

    

    View Slide

68. 68
    EXPRESSION =
    alt(
    NUMBER,
    FUNCTION_CALL,
    VARIABLE_REFERENCE,
    )
    

    View Slide

69. 69
    def eval_expr(expr)
    case expr
    when IntNode
    expr.value
    when FunCallNode
    eval_function_call(expr)
    end
    end

    

    

    View Slide

70. 70
    def eval_expr(expr)
    case expr
    when IntNode
    expr.value
    when FunCallNode
    eval_function_call(expr)
    when VarNode
    eval_var(expr)
    end
    end
    

    View Slide

71. 71
    def eval_var(expr)
    $env[expr.name]
    end
    

    View Slide

72. 72
    $env['amount'] = 5
    

    View Slide

73. Language 3.1
    73
    Variable assignment
    

    View Slide

74. %
    74
    cat stuff.cke
    4173
    amount=5
    add(2,amount)
    

    View Slide

75. %
    75
    cat stuff.cke | ./clarke
    4173
    7
    

    View Slide

76. 76
    class AssignNode < Struct.new(:name, :rhs)
    end
    

    View Slide

77. 77
    ASSIGNMENT =
    seq(
    VARIABLE_NAME,
    char('='),
    lazy { EXPRESSION },
    )
    

    View Slide

78. 78
    ASSIGNMENT =
    seq(
    VARIABLE_NAME,
    char('=').ignore,
    lazy { EXPRESSION },
    ).compact
    

    View Slide

79. 79
    ASSIGNMENT =
    seq(
    VARIABLE_NAME,
    char('=').ignore,
    lazy { EXPRESSION },
    ).compact.map do |data|
    AssignNode.new(data[0], data[1])
    end
    

    View Slide

80. 80
    EXPRESSION =
    alt(
    NUMBER,
    FUNCTION_CALL,
    VARIABLE_REFERENCE,
    )

    

    View Slide

81. 81
    EXPRESSION =
    alt(
    NUMBER,
    FUNCTION_CALL,
    VARIABLE_REFERENCE,
    ASSIGNMENT,
    )
    

    View Slide

82. 82
    def eval_expr(expr)
    case expr
    when …
    …
    when AssignNode
    eval_assign(expr)
    end
    end
    

    View Slide

83. 83
    def eval_assign(expr)
    $env[expr.name] = eval_expr(expr.rhs)
    end
    

    View Slide

84. Cleanup 3
    84
    Print function
    

    View Slide

85. 85
    def handle_success(success)
    success.data.each do |expr|
    p eval_expr(expr)
    end
    end
    

    View Slide

86. 86
    def handle_success(success)
    success.data.each do |expr|
    eval_expr(expr)
    end
    end
    

    View Slide

87. 87
    $env['print'] =
    FunDef.new(
    ['a'],
    ->(a) { p a },
    )
    

    View Slide

88. %
    88
    cat stuff.cke
    4173
    amount=5
    print(add(2,amount))
    

    View Slide

89. %
    89
    cat stuff.cke | ./clarke
    7
    

    View Slide

90. Cleanup 4
    90
    Immutable env
    

    View Slide

91. 91
    def eval_expr(expr)
    case expr
    when …
    …
    when AssignNode
    eval_assign(expr)
    end
    end
    

    View Slide

92. 92
    def eval_expr(expr, env)
    case expr
    when …
    …
    when AssignNode
    eval_assign(expr, env)
    end
    end
    

    View Slide

93. 93
    def eval_assign(expr)
    $env[expr.name] = eval_expr(expr.rhs)
    end

    

    

    View Slide

94. 94
    def eval_assign(expr, env)
    $env[expr.name] = eval_expr(expr.rhs)
    end

    

    

    View Slide

95. 95
    def eval_assign(expr, env)
    val = eval_expr(expr.rhs)
    new_env = env.merge(expr.name => val)
    ValAndEnv.new(val, new_env)
    end
    

    View Slide

96. 96
    class ValAndEnv < Struct.new(:val, :env)
    end
    

    View Slide

97. 97
    def handle_success(success)
    success.data.each do |expr|
    eval_expr(expr)
    end
    end
    

    View Slide

98. 98
    def handle_success(success)
    init = ValAndEv.new(0, INITIAL_ENV)
    

    

    

    

    View Slide

99. 99
    def handle_success(success)
    init = ValAndEv.new(0, INITIAL_ENV)
    success.data.reduce(init) do |result, e|
    eval_expr(e, result.env)
    end
    end
    

    View Slide

100. 100
     INITIAL_ENV = {
     'add' => FunDef.new(
     ['a', 'b'],
     ->(a, b) { a + b },
     )
     }
     

     View Slide

101. Language 4
     101
     Scopes
     

     View Slide

102. %
     102
     cat stuff.cke
     4173
     amount = {
     a = 2
     b = 3
     add(a, b)
     }
     print(add(2, amount))
     

     View Slide

103. %
     103
     cat stuff.cke | ./clarke
     7
     

     View Slide

104. 104
     class ScopeNode < Struct.new(:exprs)
     end
     

     View Slide

105. 105
     SCOPE =
     seq(
     char('{'),
     repeat1(lazy { EXPRESSION }),
     char('}'),
     )
     

     View Slide

106. 106
     SCOPE =
     seq(
     char('{').ignore,
     repeat1(lazy { EXPRESSION }),
     char('}').ignore,
     ).compact
     

     View Slide

107. 107
     SCOPE =
     seq(
     char('{').ignore,
     repeat1(lazy { EXPRESSION }),
     char('}').ignore,
     ).compact.map do |data|
     ScopeNode.new(data[0])
     end
     

     View Slide

108. 108
     EXPRESSION =
     alt(
     NUMBER,
     FUNCTION_CALL,
     VARIABLE_REFERENCE,
     ASSIGNMENT,
     )

     

     View Slide

109. 109
     EXPRESSION =
     alt(
     NUMBER,
     FUNCTION_CALL,
     VARIABLE_REFERENCE,
     ASSIGNMENT,
     SCOPE,
     )
     

     View Slide

110. 110
     def eval_expr(expr, env)
     case expr
     when …
     …
     when ScopeNode
     eval_scope(expr, env)
     end
     end
     

     View Slide

111. 111
     def eval_scope(expr, env)
     init = ValAndEnv.new(0, env)
     result =
     expr.exprs.reduce(init) do |result, e|
     eval_expr(e, result.env)
     end
     ValAndEnv.new(result.val, env)
     end
     

     View Slide

112. 112
     def eval_scope(expr, env)
     init = ValAndEnv.new(0, env)
     result =
     expr.exprs.reduce(init) do |result, e|
     eval_expr(e, result.env)
     end
     ValAndEnv.new(result.val, env)
     end
     

     View Slide

113. %
     113
     cat stuff.cke
     4173
     amount = {
     a = 2
     b = 3
     add(a, b)
     }
     print(add(2, amount))
     

     View Slide

114. Language 5
     114
     Conditionals
     

     View Slide

115. %
     115
     cat stuff.cke
     if (0) {
     print(100)
     } else {
     print(200)
     }
     

     View Slide

116. %
     116
     cat stuff.cke | ./clarke
     200
     

     View Slide

117. SKIP
     117
     

     View Slide

118. Language 6
     118
     Function definitions
     

     View Slide

119. %
     119
     cat stuff.cke
     fun multiply(a, b) {
     if (b) {
     add(a, multiply(a, sub(b, 1)))
     } else {
     0
     }
     }
     print(multiply(3, 5))
     

     View Slide

120. 120
     class FunDefNode < Struct.new(
     :name, :argument_names, :body)
     end
     

     View Slide

121. 121
     FUNCTION_DEF =
     seq(
     string('fun'),
     FUNCTION_NAME,
     char('('),
     intersperse(
     VARIABLE_NAME,
     char(','),
     ),
     char(')'),
     SCOPE,
     )

     

     

     View Slide

122. 122
     FUNCTION_DEF =
     seq(
     string('fun').ignore,
     FUNCTION_NAME,
     char('(').ignore,
     intersperse(
     VARIABLE_NAME,
     char(',').ignore,
     ).compact,
     char(')').ignore,
     SCOPE,
     ).compact

     

     

     View Slide

123. 123
     FUNCTION_DEF =
     seq(
     string('fun').ignore,
     FUNCTION_NAME,
     char('(').ignore,
     intersperse(
     VARIABLE_NAME,
     char(',').ignore,
     ).compact,
     char(')').ignore,
     SCOPE,
     ).compact.map do |data|
     FunDefNode.new(data[0], data[1], data[2])
     end
     

     View Slide

124. 124
     class FunDef < Struct.new(:argument_names, :body)
     end
     

     View Slide

125. View Slide

126. 125
     def eval_function_def(expr, env)
     

     View Slide

127. 125
     def eval_function_def(expr, env)
     fun_def = FunDef.new(

     expr.argument_names, expr.body)

     

     View Slide

128. 125
     def eval_function_def(expr, env)
     fun_def = FunDef.new(

     expr.argument_names, expr.body)

     ValAndEnv.new(

     fun_def,

     env.merge(expr.name => fun_def),

     )

     end
     

     View Slide

129. 126
     def eval_function_call(expr, env)
     fun = env[expr.name]
     values = expr.args.map { |e| eval_expr(e) }
     

     

     

     

     fun.body.call(*values)
     

     View Slide

130. 127
     def eval_function_call(expr, env)
     fun = env[expr.name]
     values = expr.args.map { |e| eval_expr(e) }
     case fun.body
     when Proc
     when Scope
     …
     fun.body.call(*values)
     

     View Slide

131. 127
     def eval_function_call(expr, env)
     fun = env[expr.name]
     values = expr.args.map { |e| eval_expr(e) }
     case fun.body
     when Proc
     when Scope
     …
     fun.body.call(*values)
     

     View Slide

132. View Slide

133. 128
     when Scope

     new_env = env.merge(

     Hash[fun.argument_names.zip(values)])

     

     View Slide

134. 128
     when Scope

     new_env = env.merge(

     Hash[fun.argument_names.zip(values)])

     eval_scope(fun.body, new_env)

     end
     

     View Slide

135. %
     129
     cat stuff.cke
     fun multiply(a, b) {
     if (b) {
     add(a, multiply(a, sub(b, 1)))
     } else {
     0
     }
     }
     print(multiply(3, 5))
     

     View Slide

136. Language 7
     130
     Operators
     

     View Slide

137. %
     131
     cat stuff.cke
     fun multiply(a, b) {
     if (b) {
     a + multiply(a, b - 1)
     } else {
     0
     }
     }
     print(multiply(3, 5))
     

     View Slide

138. 132
     10 - 2 - 5 * 2 + 2 ^ 3 ^ 4
     

     View Slide

139. SKIP
     133
     

     View Slide

140. etc.
     

     View Slide

141. Types! Loops! Closures! Objects!

     Classes! Inheritance! Tuples! Records!
     Typechecking! Multimethods!
     Enumerations! Pattern matching!
     Currying! Modules!
     135
     

     View Slide

142. View Slide

143. 136
     L1 Numbers
     

     View Slide

144. 136
     L1 Numbers
     L2 Functions
     

     View Slide

145. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     

     View Slide

146. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     

     View Slide

147. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     

     View Slide

148. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     C3 Print function
     

     View Slide

149. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     C3 Print function
     C4 Immutable environment
     

     View Slide

150. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     C3 Print function
     C4 Immutable environment
     L4 Scopes
     

     View Slide

151. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     C3 Print function
     C4 Immutable environment
     L4 Scopes
     L5 Conditionals
     

     View Slide

152. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     C3 Print function
     C4 Immutable environment
     L4 Scopes
     L5 Conditionals
     L6 Function definitions
     

     View Slide

153. 136
     L1 Numbers
     L2 Functions
     C1 Abstract syntax
     C2 Environment
     L3 Variables
     C3 Print function
     C4 Immutable environment
     L4 Scopes
     L5 Conditionals
     L6 Function definitions
     L7 Operators
     

     View Slide

154. 137
     github.com/ddfreyne/clarke
     

     View Slide

155. 137
     github.com/ddfreyne/clarke
     SOON
     

     View Slide

156. 138
     Ready to
     interpret
     your questions.
     DENIS@STONESHIP.ORG @DDFREYNE
     

     View Slide