$30 off During Our Annual Pro Sale. View Details »

Hy: A Lisp That Compiles to Python AST

Hy: A Lisp That Compiles to Python AST

PyCon Canada

August 11, 2013
Tweet

More Decks by PyCon Canada

Other Decks in Programming

Transcript

  1. Hy
    A Lisp the Compiles to Python... AST
    @agentultra
    http://agentultra.com
    https://www.github.com/agentultra
    Sunday, 11 August, 13

    View Slide

  2. http://goo.gl/6yPnmm
    Sunday, 11 August, 13

    View Slide

  3. http://goo.gl/6yPnmm
    Sunday, 11 August, 13

    View Slide

  4. => (import [sh [cat grep wc]])
    => (-> (cat "/usr/share/dict/words") (grep "-E" "^hy") (wc "-l"))
    210
    (-> (read) (eval) (print) (loop))
    Sunday, 11 August, 13

    View Slide

  5. Python AST
    • Incredibly useful
    • Woefully under-documented
    • Considered by many to be black magic
    • Is essentially what Python is
    Sunday, 11 August, 13

    View Slide

  6. def hello(name="World!"):
    print("Hello,", name)
    if __name__ == '__main__':
    hello("James!")
    Sunday, 11 August, 13

    View Slide

  7. Module(body=[
    FunctionDef(name='hello',
    args=arguments(args=[arg(arg='name', annotation=None)],
    vararg=None,
    varargannotation=None,
    kwonlyargs=[],
    kwarg=None,
    kwargannotation=None,
    defaults=[Str(s='World!')],
    kw_defaults=[]),
    body=[
    Expr(value=Call(func=Name(id='print', ctx=Load()),
    args=[Str(s='Hello,'), Name(id='name', ctx=Load())],
    keywords=[],
    starargs=None,
    kwargs=None))],
    decorator_list=[],
    returns=None),
    If(test=Compare(left=Name(id='__name__', ctx=Load()), ops=[Eq()],
    comparators=[Str(s='__main__')]),
    body=[
    Expr(value=Call(func=Name(id='hello', ctx=Load()),
    args=[Str(s='James!')], keywords=[], starargs=None, kwargs=None))],
    orelse=[])])
    Sunday, 11 August, 13

    View Slide

  8. http://goo.gl/iOVZt0
    Sunday, 11 August, 13

    View Slide

  9. http://goo.gl/wyri9k
    Sunday, 11 August, 13

    View Slide

  10. 1. an atom, or
    2. an expression (x . y); where x and y are s-
    expressions
    Sunday, 11 August, 13

    View Slide

  11. Sunday, 11 August, 13

    View Slide

  12. TREES!
    Sunday, 11 August, 13

    View Slide

  13. (1 . (2 . (3 . nil)))
    Sunday, 11 August, 13

    View Slide

  14. (list 1 2 3)
    (defun square (a) (* a a))
    Sunday, 11 August, 13

    View Slide

  15. eval/apply
    Sunday, 11 August, 13

    View Slide

  16. Getting Hy on Python
    Sunday, 11 August, 13

    View Slide

  17. Lexer
    Sunday, 11 August, 13

    View Slide

  18. def tokenize(buf):
    """
    Tokenize a Lisp file or string buffer into internal Hy objects.
    """
    try:
    return parser.parse(lexer.lex(buf))
    except LexingError as e:
    pos = e.getsourcepos()
    raise LexException(
    "Could not identify the next token at line %s, column %s" % (
    pos.lineno, pos.colno))
    Sunday, 11 August, 13

    View Slide

  19. from rply import LexerGenerator
    lg = LexerGenerator()
    # A regexp for something that should end a quoting/unquoting operator
    # i.e. a space or a closing brace/paren/curly
    end_quote = r'(?![\s\)\]\}])'
    lg.add('LPAREN', r'\(')
    lg.add('RPAREN', r'\)')
    lg.add('LBRACKET', r'\[')
    lg.add('RBRACKET', r'\]')
    lg.add('LCURLY', r'\{')
    lg.add('RCURLY', r'\}')
    lg.add('QUOTE', r'\'%s' % end_quote)
    lg.add('QUASIQUOTE', r'`%s' % end_quote)
    lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
    lg.add('UNQUOTE', r'~%s' % end_quote)
    lg.add('HASHBANG', r'#!.*[^\r\n]')
    lg.add('STRING', r'''(?x)
    (?:u|r|ur|ru)? # prefix
    " # start string
    (?:
    | [^"\\] # non-quote or backslash
    | \\. # or escaped single character
    | \\x[0-9a-fA-F]{2} # or escaped raw character
    | \\u[0-9a-fA-F]{4} # or unicode escape
    | \\U[0-9a-fA-F]{8} # or long unicode escape
    )* # one or more times
    " # end string
    Sunday, 11 August, 13

    View Slide

  20. Parser
    Sunday, 11 August, 13

    View Slide

  21. pg = ParserGenerator(
    [rule.name for rule in lexer.rules] + ['$end'],
    cache_id="hy_parser"
    )
    # ...
    @pg.production("paren : LPAREN list_contents RPAREN")
    @set_boundaries
    def paren(p):
    return HyExpression(p[1])
    @pg.production("paren : LPAREN RPAREN")
    @set_boundaries
    def empty_paren(p):
    return HyExpression([])
    Sunday, 11 August, 13

    View Slide

  22. Compiler
    Sunday, 11 August, 13

    View Slide

  23. In Case of Emergency
    • Lean forward
    • Place head firmly between knees
    • Scream until it goes away
    Sunday, 11 August, 13

    View Slide

  24. Problems
    Sunday, 11 August, 13

    View Slide

  25. def hy_eval(hytree, namespace, module_name):
    foo = HyObject()
    foo.start_line = 0
    foo.end_line = 0
    foo.start_column = 0
    foo.end_column = 0
    hytree.replace(foo)
    _ast, expr = hy_compile(hytree, module_name, get_expr=True)
    # Spoof the positions in the generated ast...
    for node in ast.walk(_ast):
    node.lineno = 1
    node.col_offset = 1
    for node in ast.walk(expr):
    node.lineno = 1
    node.col_offset = 1
    # Two-step eval: eval() the body of the exec call
    eval(ast_compile(_ast, "", "exec"), namespace)
    # Then eval the expression context and return that
    return eval(ast_compile(expr, "", "eval"), namespace)
    Sunday, 11 August, 13

    View Slide

  26. http://goo.gl/ce6jGN
    Sunday, 11 August, 13

    View Slide

  27. MACROS
    Sunday, 11 August, 13

    View Slide

  28. http://goo.gl/32eMO7
    Sunday, 11 August, 13

    View Slide

  29. • Parameterize your boilerplate
    • DSLs
    • Cheat
    • Program your programs
    Sunday, 11 August, 13

    View Slide

  30. def macroexpand(tree, module_name):
    if isinstance(tree, HyExpression):
    if tree == []:
    return tree
    fn = tree[0]
    if fn in ("quote", "quasiquote"):
    return tree
    ntree = HyExpression(tree[:])
    ntree.replace(tree)
    if isinstance(fn, HyString):
    m = _hy_macros[module_name].get(fn)
    if m is None:
    m = _hy_macros[None].get(fn)
    if m is not None:
    obj = _wrap_value(m(*ntree[1:]))
    obj.replace(tree)
    return obj
    return ntree
    return tree
    Sunday, 11 August, 13

    View Slide

  31. (defun princ [the-string]
    (kwapply (print the-string) {"end" ""}))
    (defn print-tag [name closingp &kwargs attrs]
    (princ "<")
    (if closingp (princ "/"))
    (princ (.lower name))
    (if (and attrs (not closingp))
    (do (princ " ")
    (princ (.join " "
    (list-comp
    (.format "{0}='{1}'" k v)
    [[k v] (.items attrs)])))))
    (print ">"))
    ;; then at the repl...
    ;; => (print-tag "body" False {"padding" 20})
    ;;
    ;; => (print-tag "body" True)
    ;;
    Sunday, 11 August, 13

    View Slide

  32. (defmacro tag [name attrs &rest body]
    (quasiquote
    (progn
    (kwapply (print-tag (unquote name) False) (unquote attrs))
    (unquote-splice (list body))
    (print-tag (unquote name) True))))
    Sunday, 11 August, 13

    View Slide

  33. (tag "body" {"padding" 20}
    (tag "h1" {"color" "red"}
    (print "The Knutsens"))
    (tag "p" {}
    (print "Lorem ipsum dolor simet")))
    ;;
    ;;
    ;; The Knutsens
    ;;
    ;;
    ;; Lorem ipsum dolor simet
    ;;
    ;;
    Sunday, 11 August, 13

    View Slide

  34. (defmacro html [&rest body]
    (quasiquote
    (tag "html" {}
    (do (unquote-splice (list body))))))
    (defmacro body [attrs &rest body]
    (quasiquote
    (tag "body" (unquote attrs)
    (unquote-splice (list body)))))
    (defmacro h1 [attrs &rest body]
    (quasiquote
    (tag "h1" (unquote attrs)
    (unquote-splice (list body)))))
    (defmacro p [attrs &rest body]
    (quasiquote
    (tag "p" (unquote attrs)
    (unquote-splice (list body)))))
    Sunday, 11 August, 13

    View Slide

  35. (html
    (body {"padding" 20}
    (p {} (print "Who the f*ck are the Knutsen's!?"))))
    Sunday, 11 August, 13

    View Slide

  36. Sunday, 11 August, 13

    View Slide

  37. Join Us
    https://github.com/hylang/hy
    http://docs.hylang.org/en/latest/
    Sunday, 11 August, 13

    View Slide

  38. Bonus Level
    Sunday, 11 August, 13

    View Slide

  39. Use ‘require’ to import
    modules with macros
    Sunday, 11 August, 13

    View Slide

  40. Interop is fun
    Sunday, 11 August, 13

    View Slide

  41. PDB works!
    Sunday, 11 August, 13

    View Slide