Hy: A Lisp That Compiles to Python AST

Hy
A Lisp the Compiles to Python... AST
@agentultra
http://agentultra.com
https://www.github.com/agentultra
Sunday, 11 August, 13

View Slide

http://goo.gl/6yPnmm

View Slide

=> (import [sh [cat grep wc]])
=> (-> (cat "/usr/share/dict/words") (grep "-E" "^hy") (wc "-l"))
210
(-> (read) (eval) (print) (loop))

View Slide

Python AST
• Incredibly useful
• Woefully under-documented
• Considered by many to be black magic
• Is essentially what Python is

View Slide

def hello(name="World!"):
print("Hello,", name)
if __name__ == '__main__':
hello("James!")

View Slide

Module(body=[
FunctionDef(name='hello',
args=arguments(args=[arg(arg='name', annotation=None)],
vararg=None,
varargannotation=None,
kwonlyargs=[],
kwarg=None,
kwargannotation=None,
defaults=[Str(s='World!')],
kw_defaults=[]),
body=[
Expr(value=Call(func=Name(id='print', ctx=Load()),
args=[Str(s='Hello,'), Name(id='name', ctx=Load())],
keywords=[],
starargs=None,
kwargs=None))],
decorator_list=[],
returns=None),
If(test=Compare(left=Name(id='__name__', ctx=Load()), ops=[Eq()],
comparators=[Str(s='__main__')]),
body=[
Expr(value=Call(func=Name(id='hello', ctx=Load()),
args=[Str(s='James!')], keywords=[], starargs=None, kwargs=None))],
orelse=[])])

View Slide

http://goo.gl/iOVZt0

View Slide

http://goo.gl/wyri9k

View Slide

1. an atom, or
2. an expression (x . y); where x and y are s-
expressions

View Slide


View Slide

TREES!

View Slide

(1 . (2 . (3 . nil)))

View Slide

(list 1 2 3)
(defun square (a) (* a a))

View Slide

eval/apply

View Slide

Getting Hy on Python

View Slide

Lexer

View Slide

def tokenize(buf):
"""
Tokenize a Lisp file or string buffer into internal Hy objects.
"""
try:
return parser.parse(lexer.lex(buf))
except LexingError as e:
pos = e.getsourcepos()
raise LexException(
"Could not identify the next token at line %s, column %s" % (
pos.lineno, pos.colno))

View Slide

from rply import LexerGenerator
lg = LexerGenerator()
# A regexp for something that should end a quoting/unquoting operator
# i.e. a space or a closing brace/paren/curly
end_quote = r'(?![\s\)\]\}])'
lg.add('LPAREN', r'$')
lg.add('RPAREN', r'$')
lg.add('LBRACKET', r'\[')
lg.add('RBRACKET', r'\]')
lg.add('LCURLY', r'\{')
lg.add('RCURLY', r'\}')
lg.add('QUOTE', r'\'%s' % end_quote)
lg.add('QUASIQUOTE', r'`%s' % end_quote)
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
lg.add('UNQUOTE', r'~%s' % end_quote)
lg.add('HASHBANG', r'#!.*[^\r\n]')
lg.add('STRING', r'''(?x)
(?:u|r|ur|ru)? # prefix
" # start string
(?:
| [^"\\] # non-quote or backslash
| \\. # or escaped single character
| \\x[0-9a-fA-F]{2} # or escaped raw character
| \\u[0-9a-fA-F]{4} # or unicode escape
| \\U[0-9a-fA-F]{8} # or long unicode escape
)* # one or more times
" # end string

View Slide

Parser

View Slide

pg = ParserGenerator(
[rule.name for rule in lexer.rules] + ['$end'],
cache_id="hy_parser"
)
# ...
@pg.production("paren : LPAREN list_contents RPAREN")
@set_boundaries
def paren(p):
return HyExpression(p[1])
@pg.production("paren : LPAREN RPAREN")
@set_boundaries
def empty_paren(p):
return HyExpression([])

View Slide

Compiler

View Slide

In Case of Emergency
• Lean forward
• Place head ﬁrmly between knees
• Scream until it goes away

View Slide

Problems

View Slide

def hy_eval(hytree, namespace, module_name):
foo = HyObject()
foo.start_line = 0
foo.end_line = 0
foo.start_column = 0
foo.end_column = 0
hytree.replace(foo)
_ast, expr = hy_compile(hytree, module_name, get_expr=True)
# Spoof the positions in the generated ast...
for node in ast.walk(_ast):
node.lineno = 1
node.col_offset = 1
for node in ast.walk(expr):
node.lineno = 1
node.col_offset = 1
# Two-step eval: eval() the body of the exec call
eval(ast_compile(_ast, "", "exec"), namespace)
# Then eval the expression context and return that
return eval(ast_compile(expr, "", "eval"), namespace)

View Slide

http://goo.gl/ce6jGN

View Slide

MACROS

View Slide

http://goo.gl/32eMO7

View Slide

• Parameterize your boilerplate
• DSLs
• Cheat
• Program your programs

View Slide

def macroexpand(tree, module_name):
if isinstance(tree, HyExpression):
if tree == []:
return tree
fn = tree[0]
if fn in ("quote", "quasiquote"):
return tree
ntree = HyExpression(tree[:])
ntree.replace(tree)
if isinstance(fn, HyString):
m = _hy_macros[module_name].get(fn)
if m is None:
m = _hy_macros[None].get(fn)
if m is not None:
obj = _wrap_value(m(*ntree[1:]))
obj.replace(tree)
return obj
return ntree
return tree

View Slide

(defun princ [the-string]
(kwapply (print the-string) {"end" ""}))
(defn print-tag [name closingp &kwargs attrs]
(princ "<")
(if closingp (princ "/"))
(princ (.lower name))
(if (and attrs (not closingp))
(do (princ " ")
(princ (.join " "
(list-comp
(.format "{0}='{1}'" k v)
[[k v] (.items attrs)])))))
(print ">"))
;; then at the repl...
;; => (print-tag "body" False {"padding" 20})
;;
;; => (print-tag "body" True)
;;

View Slide

(defmacro tag [name attrs &rest body]
(quasiquote
(progn
(kwapply (print-tag (unquote name) False) (unquote attrs))
(unquote-splice (list body))
(print-tag (unquote name) True))))

View Slide

(tag "body" {"padding" 20}
(tag "h1" {"color" "red"}
(print "The Knutsens"))
(tag "p" {}
(print "Lorem ipsum dolor simet")))
;;
;;
;; The Knutsens
;;
;;
;; Lorem ipsum dolor simet
;;
;;

View Slide

(defmacro html [&rest body]
(quasiquote
(tag "html" {}
(do (unquote-splice (list body))))))
(defmacro body [attrs &rest body]
(quasiquote
(tag "body" (unquote attrs)
(unquote-splice (list body)))))
(defmacro h1 [attrs &rest body]
(quasiquote
(tag "h1" (unquote attrs)
(defmacro p [attrs &rest body]
(quasiquote
(tag "p" (unquote attrs)

View Slide

(html
(body {"padding" 20}
(p {} (print "Who the f*ck are the Knutsen's!?"))))

View Slide


View Slide

Join Us
https://github.com/hylang/hy
http://docs.hylang.org/en/latest/

View Slide

Bonus Level

View Slide

Use ‘require’ to import
modules with macros

View Slide

Interop is fun

View Slide

PDB works!

View Slide

Hy: A Lisp That Compiles to Python AST

Hy: A Lisp That Compiles to Python AST

PyCon Canada

More Decks by PyCon Canada

Other Decks in Programming

Featured

Transcript