Осторожно, DSL! Иван Цыганов, Positive Technologies

Выступление на конференции PyCon Russia 2016

1. Осторожно, DSL! Цыганов Иван Positive Technologies
2. server { location / { proxy_pass http://localhost:8080/; } location ~ \.(gif|jpg|png)$ { root /data/images; } }
3. version: '2' services: db: image: postgres web: build: . command: python manage.py runserver 0.0.0.0:8000 volumes: - .:/code ports: - "8000:8000" depends_on: - db DOCKER COMPOSE
4. LOGROTATE compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript } /var/log/messages { rotate 5 weekly postrotate /sbin/killall -HUP syslogd endscript }
5. compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript }
6. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'compress': False,
 'sharedscripts': True,
 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript }
7. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'compress': False,
 'sharedscripts': True,
 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript } size=100k 'size': 102400,
8. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '100 KB',
9. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '100 KB', def get_size(self, value):
 size_value, _, size_unit = value.partition(' ')
 return self.units_to_bytes(size_value, size_unit)
10. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '1MB + 100KB',
11. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '1MB + 100KB', parse_size = re.compile(
 pattern=r'''
 (?P<Value>\d+)\s*
 (?P<ValueUnit>KB|MB|GB)?\s*
 ((?P<Operator>[-+/*])\s*)?
 (
 (?P<Delta>\d+)\s*
 (?P<DeltaUnit>KB|MB|GB)?
 )? ''',
 flags=re.IGNORECASE | re.VERBOSE
 ).search
12. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': ‘100MB - 2 * (1MB + 100KB)’,
13. Some people, when confronted with a problem, think "I know, I’ll use regular expressions." Now they have two problems. — Jamie Zawinski
14. Осторожно, DSL!
15. – Мартин Фаулер Предметно-ориентированный язык — это язык программирования с ограниченными выразительными возможностями, ориентированный на некую конкретную предметную область
16. ✤ SQL ✤ REGEXP ✤ TeX/LaTeX Виды DSL DSL Внутренние Внешние ✤ PonyORM ✤ WTForm ✤ Django models
17. Что будет дальше? ✤ Внутренние DSL ✤ Внешние DSL ✤ Инструменты для создания анализаторов
18. Внутренние DSL Все возможности базового языка Привычный синтаксис Ограничен базовым языком 'size': 100MB - 2 * (1MB - 100KB)
19. GB = lambda x: x*2**30
 MB = lambda x: x*2**20
 KB = lambda x: x*2**10 GB = 2**30
 MB = 2**20
 KB = 2**10 config = {
 (‘/var/log/nginx/access.log',): {
 'size': 100*MB - 2 * (1*MB - 100*KB) }, (‘/var/log/nginx/error.log',): {
 'size': 100*MB - 2 * (1*MB - 100*KB) }
 }
 'size': MB(100) - 2*(MB(1) + KB(100)) 'size': 100*MB - 2 * (1*MB - 100*KB)
20. Внешние DSL Сами выбираем синтаксис Необходимо разрабатывать анализаторы
21. compress: true
 rules:
 - files: [/var/log/nginx/access.log, /var/log/nginx/error.log]
 rotate: 5
 size: 100MB - 2 * (1MB + 100KB)
 postrotate: [/sbin/killall -HUP syslogd]
 config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 100*MB - 2 * (1*MB - 100*KB) 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }

22. Python Lex-Yacc (PLY)
23. ply.lex ply.yacc AST Run Code source
24. ply.lex Type Value PLUS + MINUS - MUL * DIV / UNIT GB|MB|KB|B DIGIT \d+ LPAREN ( RPAREN )
25. ply.lex 2 * (1KB + 1KB) DIGIT = 2 MUL = * LPAREN = ( DIGIT = 1 UNIT = KB PLUS = + DIGIT = 1 UNIT = KB RPAREN = )
26. ply.lex 1KB1KB-MB DIGIT = 1 DIGIT = 1 UNIT = KB MINUS = - UNIT = MB UNIT = KB
27. ply.yacc expression : expression PLUS expression
 | expression MINUS expression
 | expression MUL expression
 | expression DIV expression expression : LPAREN expression RPAREN expression : DIGIT UNIT expression : DIGIT
28. ply.yacc def p_expression(self, p):
 '''
 expression : expression PLUS expression
 | expression MINUS expression
 | expression MUL expression
 | expression DIV expression
 '''
 if p[2] == '+': p[0] = p[1] + p[3]
 if p[2] == '-': p[0] = p[1] - p[3]
 if p[2] == '*': p[0] = p[1] * p[3]
 if p[2] == '/': p[0] = p[1] / p[3]
29. ply.yacc precedence = (
 ('left', 'PLUS', 'MINUS'),
 ('left', 'MUL', 'DIV'),
 ) def p_expression(self, p):
 '''
 expression : expression PLUS expression
 | expression MINUS expression
 | expression MUL expression
 | expression DIV expression
 '''
 if p[2] == '+': p[0] = p[1] + p[3]
 if p[2] == '-': p[0] = p[1] - p[3]
 if p[2] == '*': p[0] = p[1] * p[3]
 if p[2] == '/': p[0] = p[1] / p[3]
30. PLY Гибкость Отладка Обработка ошибок Понятный код библиотеки Высокий порог входа Многословный
31. funcparserlib
32. def tokenize(str):
 specs = [
 ('Spaces', (r'[ \s\t\r\n]+',)),
 
 ('PLUS', (r'\+',)),
 ('MINUS', (r'-',)),
 ('MUL', (r'\*',)),
 ('DIV', (r'/',)),
 ('UNIT', (r'GB|MB|KB|B',)),
 ('DIGIT', (r'\d+',)),
 ('LPAREN', (r'\(',)),
 ('RPAREN', (r'\)',)),
 ]
 return list(filter(
 lambda t: t.type != 'Spaces',
 (t for t in make_tokenizer(specs)(str))
 )) funcparserlib
33. number = value_of('DIGIT') >> int
 unit = number + value_of('UNIT') >> to_bytes
 operand = unit | number
 
 makeop = lambda s, f: skip_token(s) >> const(f)
 add = makeop('PLUS', operator.add)
 sub = makeop('MINUS', operator.sub)
 mul = makeop('MUL', operator.mul)
 div = makeop('DIV', operator.floordiv) 
 mul_op = mul | div
 add_op = add | sub funcparserlib
34. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib
35. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2
36. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 maybe(expr)
37. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 term
38. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 primary
39. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 2 operand
40. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 2 many(mul_op + primary)
41. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib many(add_op + term) 2+2*2 2+
42. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib primary 2+2*2 2+
43. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib operand 2+2*2 2+2
44. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 2+2* many(mul_op + primary)
45. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib operand 2+2*2 2+2*2
46. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib eval_expr 2+2*2 2+2*2
47. 2+4 primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib eval_expr
48. 6 primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib maybe(expr)
49. funcparserlib Компактный Гибкий Для любителей функционального программирования :) Многое приходится делать руками Для любителей функционального программирования :)
50. pyparsing
51. pyparsing plusop = oneOf('+ -')
 multop = oneOf('* /')
 
 digit = Word(nums)
 unit = digit + oneOf('GB MB KB’) operand = unit | digit
 
 parser = Forward()
 primary = operand | Literal('(') + parser + Literal(')')
 term = (primary + ZeroOrMore(multop + primary)).setParseAction(eval_expr)
 expr = (term + ZeroOrMore(plusop + term)).setParseAction(eval_expr)
 parser << Optional(expr)

52. pyparsing plusop = oneOf('+ -')
 multop = oneOf('* /')
 digit = Word(nums)
 unit = digit + oneOf('GB MB KB’) operand = unit | digit 
 parser = operatorPrecedence(
 operand, [
 (multop, 2, opAssoc.LEFT, calculate),
 (plusop, 2, opAssoc.LEFT, calculate)
 ]
 )
53. pyparsing Понятный код Базовые компоненты Свои компоненты Документация Отладка
54. А что насчет быстродействия?
55. Скорость
56. Скорость
57. А что же пользователи?
58. Сообщения об ошибках Traceback (most recent call last): File "size_parser/on_ply.py", line 100, in parse return parser.parse(string) File "size_parser/on_ply.py", line 94, in parse p = self._parser.parse(data, debug=self._debug) File ".env/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack lookahead = get_token() # Get the next token File ".env/site-packages/ply/lex.py", line 396, in token raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) ply.lex.LexError: Illegal character '_' at index 5
59. Traceback (most recent call last): File "size_parser/on_ply.py", line 100, in parse return parser.parse(string) File "size_parser/on_ply.py", line 94, in parse p = self._parser.parse(data, debug=self._debug) File ".env/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack lookahead = get_token() # Get the next token File ".env/site-packages/ply/lex.py", line 396, in token raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) ply.lex.LexError: Illegal character '_' at index 5 Сообщения об ошибках
60. ply 
 def t_error(self, t):
 raise ParserException(t, self.source)
 
 def p_error(self, p):
 raise ParserException(p, self.source) >>> parse(‘1MB+_GB-100KB’) Unexpected "_GB-100KB" at position 4: 1MB+_GB-100KB ^^^
61. Так что же выбрать?
62. Что же выбрать? pyparsing ✤ Хочу легко все описать ✤ Быстродействие не главное
63. Что же выбрать? funcparserlib pyparsing ✤ Хочу легко все описать ✤ Быстродействие не главное ✤ Люблю функциональное программирование ✤ Быстродействие не главное
64. Что же выбрать? PLY funcparserlib pyparsing ✤ Хочу как в учебнике ✤ Скорость работы - главное! ✤ Хочу легко все описать ✤ Быстродействие не главное ✤ Люблю функциональное программирование ✤ Быстродействие не главное
65. И что в итоге? ✤ Для простых задач попробуйте: ✤ Средства самого языка ✤ Регулярные выражения ✤ Если задача сложная: ✤ Внутренние DSL ✤ Yaml, Json, XML, … ✤ Внешние DSL
66. Спасибо за внимание! Вопросы? http://tsyganov-ivan.com/