玩转 AST

玩转 AST
构建自己的代码分析和代码重写工具
黄志敏
@flyerhzm

View Slide

什么是 AST
？
抽象语法树（Abstract Syntax Tree
）

View Slide

class RubyChinaConf
def location
'Shanghai'
end
def year
2023
end
end
(class
(const nil :RubyChinaConf) nil
(begin
(def :location
(args)
(str "Shanghai"))
(def :year
(args)
(int 2023))))

View Slide

View Slide

AST
的用途
编译器
解释器
代码静态分析工具

View Slide

哪些 RUBY
工具使用 AST
？

View Slide

FORMATTER
如何工作？

View Slide

class RubyChinaConf
# body
end

View Slide

class RubyChinaConf
def location
'Shanghai'
end
end

View Slide

class RubyChinaConf
def location
'Shanghai'
end
def year
2023
end
end

View Slide

PRETTIER
[
group([
group(["class ", "RubyChinaConf"]),
indent([
hardline,
group([
group([
group(["def ", "location", group([])]),
indent([hardline, group(["'", "Shanghai", "'"])]),
hardline,
"end",
]),
hardline,
hardline,
group([
group(["def ", "year", group([])]),
indent([hardline, group("2023")]),
hardline,
"end",
]),
]),
]),
hardline,
"end",
]),
hardline,
]

View Slide

foobar = { foo: foo, bar: bar }
group([
"foobar",
" = ",
group([
"{",
indent([
line,
group(["foo:", indent([line, "foo"])]),
",",
line,
group(["bar:", indent([line, "bar"])]),
]),
line,
"}",
]),
])

View Slide

LINTER
如何工作？

View Slide

案例
rails_best_practices
synvert

View Slide

RAILS_BEST_PRACTICES

View Slide

记录 MODEL
和 ASSOCIATION
# app/models/*.rb
add_callback :start_class do |node|
@klass = node.class_name.to_s
end
add_callback :start_command do |node|
if node.message.to_s == 'belongs_to'
association_name = node.arguments.all.first.to_s
add_association(@klass, association_name)
end
end

View Slide

记录 TABLE
和 INDEX
# db/schema.rb
add_callback :start_command do |node|
if node.message.to_s == 'add_index'
table_name = node.arguments.all.first.to_s
index_column = node.arguments.all[1].to_object
@index_columns[table_name] << index_column
end
end

View Slide

检查遗漏的 INDEX
add_callback :after_check do
associations.each do |klass, association_name|
unless @index_columns[klass.table_name].include?(association_name.column_name)
add_error "always add db index (#{klass.table_name} => #{association_name.column_name})",
table_node.file, table_node.line_number
end
end
end

View Slide

SYNVERT
Synvert::Rewriter.new 'factory_bot', 'use_short_syntax' do
within_files '**/*.rb' do
find_node '.send[receiver=FactoryBot][message in (create build build_stubbed attributes_for)]' do
delete :receiver, :dot
end
end
end

View Slide

源代码转换成 AST
------------
| AST Node |
------------
/|\
|
---------------
| Source Code |
---------------

View Slide

AST NODE
包含
node type
children
location (start, end, line, column)
source

View Slide

增加 ATTRIBUTE
ripper + syntax_tree + syntax_tree_ext
parser + parser_node_ext
code = 'FactoryBot.create(:user)'
node = Parser::CurrentRuby.parse(code)
node.children[0]
node.receiver
node.children[1]
node.message

View Slide

查找 AST NODE
https://github.com/xinminlabs/node-query-ruby
--------------
| Node Query |
--------------
/|\
|
------------
| AST Node |
------------
/|\
|
---------------
| Source Code |
---------------

View Slide

CSS
/* HTML */

/* CSS */
.item[data-type='image']

View Slide

NQL (NODE QUERY LANGUAGE)
# Ruby Code
FactoryBot.create(:user)
# AST Node
(send
(const nil :FactoryBot) :create
(sym :user))
# NQL
.send[receiver=FactoryBot][message=create]

View Slide

REX & RACC (LEX & YACC)

View Slide

解析 NODE TYPE
.send

View Slide

class NodeQueryLexer
macros
NODE_TYPE /\.[a-zA-Z]+/
rules
# [:state] pattern [actions]
/\s+/
/#{NODE_TYPE}/ { [:tNODE_TYPE, text[1..]] }
end

View Slide

class NodeQueryParser
token tNODE_TYPE
rule
basic_selector
: tNODE_TYPE { Compiler::BasicSelector.new(node_type: val[0]) }
end
end

View Slide

module Compiler
class BasicSelector
def initialize(node_type:)
@node_type = node_type
end
def query_nodes(node)
# iterate all child nodes,
# check if child node matches node_type
end
def match?(node)
@node_type.to_sym == NodeQuery.adapter.get_node_type(node)
end
end
end
selector = NodeQueryParser.new.parse(nql)
selector.query_nodes(node) # matching_nodes

View Slide

解析 ATTRIBUTE
.send[reciever=FactoryBot]

View Slide

OPEN_ATTRIBUTE /\[/
CLOSE_ATTRIBUTE /\]/
IDENTIFIER /[@\*\-\.\w]*\w/
IDENTIFIER_VALUE /[@\.\w!&:\?<>=]+/
rules
/#{OPEN_ATTRIBUTE}/ { @state = :KEY; [:tOPEN_ATTRIBUTE, text] }
:KEY /\s+/
:KEY /=/ { @state = :VALUE; [:tOPERATOR, '='] }
:KEY /#{IDENTIFIER}/ { [:tKEY, text] }
:VALUE /\s+/
:VALUE /#{CLOSE_ATTRIBUTE}/ { @state = nil; [:tCLOSE_ATTRIBUTE, text] }
:VALUE /#{IDENTIFIER_VALUE}/ { [:tIDENTIFIER_VALUE, text] }
end
1
macros
2
3
4
5
6
7
8
9
/\s+/
10
11
12
13
14
15
16
17
18
19

View Slide

token tNODE_TYPE tKEY tIDENTIFIER_VALUE tOPEN_ATTRIBUTE tCLOSE_ATTRIBUTE tOPERATOR
| tNODE_TYPE attribute { Compiler::BasicSelector.new(node_type: val[0], attribute: val[1]) }
attribute
: tOPEN_ATTRIBUTE tKEY tOPERATOR value tCLOSE_ATTRIBUTE { Compiler::Attribute.new(key: val[1], value: val[3], opera
value
: tIDENTIFIER_VALUE { Compiler::Identifier.new(value: val[0]) }
end
1
2
rule
3
basic_selector
4
5
6
7
8
9
10
11
12
13
end
14

View Slide

module Compiler
class BasicSelector
def initialize(node_type:, attribute: nil)
@node_type = node_type
@attribute = attribute
end
def query_nodes(node)
# iterate all child nodes,
# check if child node matches node_type and attribute
end
def match?(node)
@node_type.to_sym == NodeQuery.adapter.get_node_type(node) && (!@attribute || @attribute.match?(node))
end
end
class Attribute
def initialize(key:, value:, operator: '=')
@key = key
@value = value
@operator = operator
end
def match?(node)
@value.match?(NodeQuery::Helper.get_target_node(node, @key), node, @operator)
end
end
end
selector = NodeQueryParser.new.parse(nql)
selector.query_nodes(node) # matching_nodes

View Slide

解析多个 ATTRIBUTE
.send[reciever=FactoryBot][message=create]

View Slide

| tNODE_TYPE attribute_list { Compiler::BasicSelector.new(node_type: val[0], attribute_list: val[1]) }
attribute_list
: attribute attribute_list { Compiler::AttributeList.new(attribute: val[0], rest: val[1]) }
| attribute { Compiler::AttributeList.new(attribute: val[0]) }
1
2
rule
3
basic_selector
4
5
6
7
8
9
10
11
attribute
12
13
14
value
15
16
end
17
end
18

View Slide

解析 VALUE
类型
.send[arguments.first=:user][arguments.last=true]

View Slide

FALSE /false/
FLOAT /\-?\d+\.\d+/
INTEGER /\-?\d+/
NIL /nil/
SYMBOL /:[\w!\?<>=]+/
TRUE /true/
SINGLE_QUOTE_STRING /'.*?'/
DOUBLE_QUOTE_STRING /".*?"/
rules
:VALUE /#{NIL}/ { [:tNIL, nil] }
:VALUE /#{TRUE}/ { [:tBOOLEAN, true] }
:VALUE /#{FALSE}/ { [:tBOOLEAN, false] }
:VALUE /#{SYMBOL}/ { [:tSYMBOL, text[1..-1].to_sym] }
:VALUE /#{FLOAT}/ { [:tFLOAT, text.to_f] }
:VALUE /#{INTEGER}/ { [:tINTEGER, text.to_i] }
:VALUE /#{DOUBLE_QUOTE_STRING}/ { [:tSTRING, text[1...-1]] }
:VALUE /#{SINGLE_QUOTE_STRING}/ { [:tSTRING, text[1...-1]] }
1
macros
2
3
OPEN_ATTRIBUTE /\[/
4
CLOSE_ATTRIBUTE /\]/
5
IDENTIFIER /[@\*\-\.\w]*\w/
6
IDENTIFIER_VALUE /[@\.\w!&:\?<>=]+/
7
8
9
10
11
12
13
14
15
16
17
/\s+/
18
19
/#{OPEN_ATTRIBUTE}/ { @state = :KEY; [:tOPEN_ATTRIBUTE, text] }
20
:KEY /\s+/
21
:KEY /=/ { @state = :VALUE; [:tOPERATOR, '='] }
22
:KEY /#{IDENTIFIER}/ { [:tKEY, text] }
23
:VALUE /\s+/
24
:VALUE /#{CLOSE_ATTRIBUTE}/ { @state = nil; [:tCLOSE_ATTRIBUTE, text] }
25
26
27
28
29
30
31
32
33
:VALUE /#{IDENTIFIER_VALUE}/ { [:tIDENTIFIER_VALUE, text] }
34

View Slide

end
35

View Slide

tBOOLEAN tFLOAT tINTEGER tNIL tSTRING tSYMBOL
| tBOOLEAN { Compiler::Boolean.new(value: val[0]) }
| tFLOAT { Compiler::Float.new(value: val[0]) }
| tINTEGER { Compiler::Integer.new(value: val[0])}
| tNIL { Compiler::Nil.new(value: val[0]) }
| tSTRING { Compiler::String.new(value: val[0]) }
| tSYMBOL { Compiler::Symbol.new(value: val[0]) }
end
1
2
3
rule
4
basic_selector
5
6
| tNODE_TYPE attribute_list { Compiler::BasicSelector.new(node_type: val[0], attribute_list: val[1]) }
7
8
attribute_list
9
: attribute attribute_list { Compiler::AttributeList.new(attribute: val[0], rest: val[1]) }
10
| attribute { Compiler::AttributeList.new(attribute: val[0]) }
11
12
attribute
13
14
15
value
16
17
18
19
20
21
22
23
24
end
25

View Slide

解析操作符
.send[arguments.size>0]

View Slide

macros
OPERATOR /(\^=|\$=|\*=|!=|=~|!~|>=|<=|>|<|=|not includes|includes|not in|in)/i
rules
# :KEY /=/ { @state = :VALUE; [:tOPERATOR, '='] }
:KEY /#{OPERATOR}/ { @state = :VALUE; [:tOPERATOR, text] }
end

View Slide

查找 DEBUG
代码
puts "hello world"
p current_user
# NQL
'.send[receiver=nil][message IN (puts p)]'

View Slide

查找 HASH ROCKET
的 KEY
{ :foo => 'bar' }
# NQL
%q{.pair[key=.sym][key=~/\A:([^'"]+)\z/]}

View Slide

查找 HASH
的键值对是相同的
a: a,
c: c,
some_method(
1
2
b: b + 2,
3
4
d: d + 4
5
)
6
# NQL
'.pair[key="{{value}}"]'

View Slide

查找 MINITEST SETUP
方法中没有调用 SUPER
def setup
do_something
end
class TestMeme < Minitest::Test
1
2
3
4
end
5
# NQL
'.class[parent_class=Minitest::Test] .def[name=setup]:not_has(> .super)'

View Slide

查找 HASH
的值
render nothing: true, status: :created
render nothing: true
1
2
# NQL
'.hash[status_value!=nil]'

View Slide

替换源代码
https://github.com/xinminlabs/node-mutation-ruby
-------------- -----------------
| Node Query | | Node Mutation |
-------------- -----------------
|\¯¯ ¯¯/|
\ /
------------
| AST Node |
------------
/|\
|
---------------
| Source Code |
---------------

View Slide

REPLACE_WITH
errors[:base] = "author not present"
1
2
errors.add(:base, "author not present")
3
replace_with node, '{{receiver}}.add({{arguments.0}}, {{arguments.1}})'

View Slide

REPLACE_WITH
1
2
3 errors.add(:base, "author not present")
1
2
3

View Slide

REPLACE_WITH
1
2
3 errors.add(:base, "author not present")
1
2
3
[{
start: 0,
end: 'errors[:base] = "author not present"'.length,
new_code: '{{receiver}}.add({{arguments.0}}, {{arguments.1}})'
}]

View Slide

REPLACE
class Post < ActiveRecord::Base
end
1
2
3
class Post < ApplicationRecord
4
end
5
replace node, :parent_class, with: 'ApplicationRecord'

View Slide

REPLACE
end
1
2
3
4
end
5
end
1
end
2
3
4
5

View Slide

REPLACE
end
1
2
3
4
end
5
end
1
end
2
3
4
5
[{
start: 'class Post < '.length,
end: 'class Post < ActiveRecord::Base'.length,
new_code: 'ApplicationRecord'
}]

View Slide

INSERT
open('http://test.com')
1
2
URI.open('http://test.com')
3
insert node, 'URI.', at: 'beginning'

View Slide

INSERT
1
2
3 URI.open('http://test.com')
1
2
3

View Slide

INSERT
1
2
3 URI.open('http://test.com')
1
2
3
[{
start: 0,
end: 0,
new_code: 'URI.'
}]

View Slide

PREPEND
def setup
do_something
end
1
2
3
4
end
5
6
7
def setup
8
super
9
do_something
10
end
11
end
12
prepend node, 'super'

View Slide

PREPEND
def setup
do_something
end
1
2
3
4
end
5
6
7
def setup
8
super
9
do_something
10
end
11
end
12
def setup
super
do_something
end
1
def setup
2
do_something
3
end
4
end
5
6
7
8
9
10
11
end
12

View Slide

PREPEND
def setup
do_something
end
1
2
3
4
end
5
6
7
def setup
8
super
9
do_something
10
end
11
end
12
def setup
super
do_something
end
1
def setup
2
do_something
3
end
4
end
5
6
7
8
9
10
11
end
12
[{
start: "class TestMeme < Minitest::Test\n def setup\n ".length,
end: "class TestMeme < Minitest::Test\n def setup\n ".length,
new_code: "super\n "
}]

View Slide

APPEND
def teardown
clean_something
end
1
2
3
4
end
5
6
7
def teardown
8
clean_something
9
super
10
end
11
end
12
append node, 'super'

View Slide

APPEND
def teardown
clean_something
end
1
2
3
4
end
5
6
7
def teardown
8
clean_something
9
super
10
end
11
end
12
def teardown
clean_something
super
end
1
def teardown
2
clean_something
3
end
4
end
5
6
7
8
9
10
11
end
12

View Slide

APPEND
def teardown
clean_something
end
1
2
3
4
end
5
6
7
def teardown
8
clean_something
9
super
10
end
11
end
12
def teardown
clean_something
super
end
1
def teardown
2
clean_something
3
end
4
end
5
6
7
8
9
10
11
end
12
[{
start: "class TestMeme < Minitest::Test\n def teardown\n clean_someting".length,
end: "class TestMeme < Minitest::Test\n def teardown\n clean_someting".length,
new_code: "\n super"
}]

View Slide

DELETE
1
2
create(:user)
3
delete node, :receiver, :dot

View Slide

DELETE
1
2
create(:user)
3 create(:user)
1
2
3

View Slide

DELETE
1
2
create(:user)
3 create(:user)
1
2
3
[{
start: 0,
end: 'FactoryBot.'.length,
new_code: ''
}]

View Slide

REMOVE
puts(
'hello'\
'world'
)
1
2
3
4
5
# nothing
6
remove node

View Slide

REMOVE
puts(
'hello'\
'world'
)
1
2
3
4
5
# nothing
6 # nothing
puts(
1
'hello'\
2
'world'
3
)
4
5
6
remove node

View Slide

REMOVE
puts(
'hello'\
'world'
)
1
2
3
4
5
# nothing
6 # nothing
puts(
1
'hello'\
2
'world'
3
)
4
5
6
remove node
[{
start: 0,
end: "puts(\n 'hello'\\\n 'world'\n)".length,
new_code: ''
}]

View Slide

HASH NODE
after_commit :add_to_index_later, on: :create
1
2
after_create_commit :add_to_index_later
3
replace node, :message, with: 'after_{{arguments.-1.on_value.to_value}}_commit'
delete node, 'arguments.-1.on_pair', and_comma: true

View Slide

HASH NODE
1
2
3 after_create_commit :add_to_index_later
1
2
3

View Slide

HASH NODE
1
2
3 after_create_commit :add_to_index_later
1
2
3
[{
start: 0,
end: 'after_commit'.length,
new_code: 'after_{{arguments.-1.on_value.to_value}}_commit'
}, {
start: 'after_commit :add_to_index_later'.length,
end: 'after_commit :add_to_index_later, :on: :create'.length,
new_code: ''
}]

View Slide

查找 +
替换
require 'parser/current'
require 'parser_node_ext'
require 'node_query'
require 'node_mutation'
1
2
3
4
5
source = "foo\nputs foo\n"
6
node = Parser::CurrentRuby.parse(source)
7
query = NodeQuery.new('.send[receiver=nil][message IN (p puts)]')
8
mutation = NodeMutation.new(source)
9
query.query_nodes(node).each do |matching_node|
10
mutation.remove(matching_node)
11
end
12
result = mutation.process
13
result.new_source # foo\n
14

View Slide

查找 +
替换
1
2
3
4
5
6
7
8
9
10
11
end
12
13
14
end
1
2
3
4
5
6
7
8
9
10
11
12
13
14

View Slide

查找 +
替换
1
2
3
4
5
6
7
8
9
10
11
end
12
13
14
end
1
2
3
4
5
6
7
8
9
10
11
12
13
14
1
2
3
4
5
6
7
8
9
10
11
end
12
13
14

View Slide

SYNVERT
代码升级工具（比如 rails
）
代码自动重写工具
基于 AST
的查找替换工具
https://synvert.net/

View Slide

-------------------- --------------- .
| Synvert Snippets | --- | Synvert CLI | .
-------------------- --------------- .
/|\
|
----------------
| Synvert Core |
----------------
¯¯/| |\¯¯
/ \
-------------- -----------------
-------------- -----------------
|\¯¯ ¯¯/|
\ /
------------
| AST Node |
------------
/|\
|
---------------
| Source Code |
---------------

View Slide

Synvert::Rewriter.new 'ruby', 'remove_debug_code' do
within_files Synvert::ALL_RUBY_FILES do
find_node '.send[receiver=nil][message IN (puts p)]' do
remove
end
end
end

View Slide

Synvert::Rewriter.new 'rails', 'convert_dynamic_finders' do
configure(parser: Synvert::PARSER_PARSER)
helper_method :dynamic_finder_to_hash do |prefix|
# convert 'email_and_active(email, true)' to 'email: email, active: true'
end
if_gem 'rails', '>= 3.0'
within_files Synvert::ALL_RUBY_FILES + Synvert::ALL_RAKE_FILES do
# find_all_by_email_and_active(email, true) => where(email: email, active: true)
find_node '.send[message=~/^find_all_by_/]' do
hash_params = dynamic_finder_to_hash('find_all_by_')
if hash_params
replace :message, with: 'where'
replace :arguments, with: hash_params
end
end

View Slide

# find_last_by_email_and_active(email, true) => where(email: email, active: true).last
find_node '.send[message=~/^find_last_by_/]' do
hash_params = dynamic_finder_to_hash('find_last_by_')
if hash_params
replace :message, with: 'where'
insert '.last', at: 'end'
end
end
# find_by_email_and_active(email, true) => find_by(email: email, active: true)
find_node '.send[message=~/^find_by_/]' do
if :find_by_id == node.message
replace :message, with: 'find_by'
replace :arguments, with: 'id: {{arguments}}'
elsif :find_by_sql != node.message
hash_params = dynamic_finder_to_hash('find_by_')
if hash_params
replace :message, with: 'find_by'
end
end
end
end
end

View Slide

文件处理
add_file 'app/models/application_record.rb', <<~EOS
class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true
end
EOS
remove_file 'config/initializers/secret_token.rb'

View Slide

SNIPPET
组合
add_snippet 'minitest', 'assert_empty'
add_snippet 'minitest/assert_instance_of'
add_snippet '/Users/flyerhzm/.synvert-ruby/lib/minitest/assert_match.rb'
add_snippet 'https://github.com/xinminlabs/synvert-snippets-ruby/blob/main/lib/minitest/assert_silent.rb'

View Slide

SYNVERT
命令行
synvert-ruby --run rspec/use_new_syntax
synvert-ruby --run ~/.synvert-ruby/lib/rspec/use_new_syntax.rb
synvert-ruby --run https://raw.githubusercontent.com/xinminlabs/synvert-snippets-ruby/master/lib/rspec/use_new_syntax.rb

View Slide

--------------------------------------------------------------
| ---------------------- --------------- --------------- |
| | Synvert VSCode Ext | | Synvert GUI | | Synvert Web | |
| ---------------------- --------------- --------------- |
--------------------------------------------------------------
/|\ /|\
| |
-------------------- --------------- ---------------
| Synvert Snippets | --- | Synvert CLI | | Synvert API |
-------------------- --------------- ---------------
/|\
|
----------------
| Synvert Core |
----------------
¯¯/| |\¯¯
/ \
-------------- -----------------
-------------- -----------------
|\¯¯ ¯¯/|
\ /
---------------
| AST |
---------------
/|\
|
---------------
| Source Code |
---------------

View Slide

SYNVERT GUI

View Slide

运行 SNIPPET

View Slide

自动生成 SNIPPET

View Slide

显示 SNIPPETS

View Slide

View Slide

演示

View Slide

View Slide

SYNVERT WEB (PLAYGROUND)

View Slide

SYNVERT JAVASCRIPT
new Synvert.Rewriter("typescript", "array-type", () => {
configure({ parser: Synvert.Parser.TYPESCRIPT });
// const z: Array = ['a', 'b'];
// =>
// const z: (string|number)[] = ['a', 'b'];
withinFiles(Synvert.ALL_TS_FILES, function () {
findNode(
`.TypeReference
[typeName.escapedText=Array]
[typeArguments.0=.UnionType]`,
() => {
replaceWith("({{typeArguments}})[]");
}
);
});
});

View Slide

CSS/LESS/SASS/SCSS
new Synvert.Rewriter("sass", "convert-to-scss", () => {
configure({ parser: Synvert.Parser.GONZALES_PE });
withinFiles(Synvert.ALL_SASS_FILES, function () {
findNode(".atrule .string", () => { insert(";", { at: "end" }); });
findNode(".declaration, .include", () => { insert(";", { at: "end", conflictPosition: -99 }); });
findNode(".mixin .operator", () => { replaceWith("@mixin "); });
findNode(".include .operator", () => { replaceWith("@include "); });
findNode(".mixin", () => {
const conflictPosition = -this.currentNode.start.column;
insert(" {", { at: "end", to: "arguments" });
insertAfter("}", {
to: "block",
newLinePosition: "after",
conflictPosition,
});
});
findNode(".ruleset", () => {
const conflictPosition = -this.currentNode.start.column;
insert(" {", { at: "end", to: "selector" });
insertAfter("}", {
to: "block",
newLinePosition: "before",
conflictPosition,
});
});

View Slide

});
renameFile(Synvert.ALL_SASS_FILES, (filePath) => filePath.replace(/\.sass$/, ".scss"));
});

View Slide

联系我们
https://synvert.net/contact_us

View Slide

谢谢
提问？

View Slide

View Slide

玩转 AST

玩转 AST

flyerhzm

More Decks by flyerhzm

Other Decks in Technology

Featured

Transcript