How to write your own Go tool

B1019ca5714cf8e9951868d6bc517827?s=47 Fatih Arslan
November 18, 2016

How to write your own Go tool

Go tools are very powerful and yet simple to use. But how are Go tools created? In this talk I’m going to answer this question by showing the various Go parser family packages (go/token, go/scanner, go/parser, etc…) and how to use them to create your own Go tool from scratch.

B1019ca5714cf8e9951868d6bc517827?s=128

Fatih Arslan

November 18, 2016
Tweet

Transcript

  1. 5.

    Some of features in motion: • Jump to function or

    type declaration • Jump to next or previous function declaration • Select function content (delete/copy/change...) • ...
  2. 10.

    go/token Package token defines constants representing the lexical tokens of

    the Go programming language and basic operations on tokens (printing, predicates).
  3. 11.
  4. 12.
  5. 13.
  6. 15.
  7. 16.
  8. 17.

    go/scanner Package scanner implements a scanner for Go source text.

    It takes a []byte as source which can then be tokenized through repeated calls to the Scan method.
  9. 18.

    var sum int = 3 + 2 Run through a

    scanner go/scanner token.VAR token.IDENT token.EOF . . .
  10. 20.

    func main() { src := []byte(`var sum int = 3

    + 2`) // Initialize the scanner. var s scanner.Scanner fset := token.NewFileSet() file := fset.AddFile("", fset.Base(), len(src)) s.Init(file, src, nil, scanner.ScanComments) // run the scanner for { pos, tok, lit := s.Scan() if tok == token.EOF { break } fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit) } }
  11. 21.

    func main() { src := []byte(`var sum int = 3

    + 2`) // Initialize the scanner. var s scanner.Scanner fset := token.NewFileSet() file := fset.AddFile("", fset.Base(), len(src)) s.Init(file, src, nil, scanner.ScanComments) // run the scanner for { pos, tok, lit := s.Scan() if tok == token.EOF { break } fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit) } }
  12. 22.

    func main() { src := []byte(`var sum int = 3

    + 2`) // Initialize the scanner. var s scanner.Scanner fset := token.NewFileSet() file := fset.AddFile("", fset.Base(), len(src)) s.Init(file, src, nil, scanner.ScanComments) // run the scanner for { pos, tok, lit := s.Scan() if tok == token.EOF { break } fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit) } }
  13. 24.

    var sum int = 3 + 2 Pos Token Literal

    1:1 VAR "var" 1:5 IDENT "sum" 1:9 IDENT "int" 1:13 ASSIGN "=" 1:15 INT "3" 1:19 INT "2" 1:17 ADD "+" 1:20 EOF "" go/scanner
  14. 26.
  15. 38.
  16. 43.

    var sum int = 3 + 2 *ast.ValueSpec *ast.Ident *ast.Ident

    *ast.BinaryExpr *ast.BasicLit *ast.BasicLit sum int 3 2 + var =
  17. 44.

    go/parser Package parser implements a parser for Go source files.

    Input may be provided in a variety of forms; the output is an abstract syntax tree (AST) representing the Go source.
  18. 45.

    var sum int = 3 + 2 go/parser *ast.ValueSpec *ast.Ident

    *ast.Ident *ast.BinaryExpr *ast.BasicLit *ast.BasicLit
  19. 46.

    var sum int = 3 + 2 go/parser *ast.ValueSpec *ast.Ident

    *ast.Ident *ast.BinaryExpr *ast.BasicLit *ast.BasicLit go/scanner go/token
  20. 48.

    func main() { src := []byte(`package main var sum int

    = 3 + 2`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } ast.Fprint(os.Stdout, fset, node, nil) }
  21. 49.

    func main() { src := []byte(`package main var sum int

    = 3 + 2`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } ast.Fprint(os.Stdout, fset, node, nil) }
  22. 50.

    func main() { src := []byte(`package main var sum int

    = 3 + 2`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } ast.Fprint(os.Stdout, fset, node, nil) }
  23. 51.
  24. 53.

    Let us find a line and column number package main

    import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) }
  25. 54.

    package main import "fmt" func main() { var sum int

    = 3 + 2 fmt.Println(sum) } *ast.BinaryExpr Let us find a line and column number
  26. 55.

    src := []byte(`package main import "fmt" func main() { var

    sum int = 3 + 2 fmt.Println(sum) }`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } // start searching for the main() function declaration for _, decl := range node.Decls { ...
  27. 56.

    src := []byte(`package main import "fmt" func main() { var

    sum int = 3 + 2 fmt.Println(sum) }`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } // start searching for the main() function declaration for _, decl := range node.Decls { ...
  28. 57.

    src := []byte(`package main import "fmt" func main() { var

    sum int = 3 + 2 fmt.Println(sum) }`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } // start searching for the main() function declaration for _, decl := range node.Decls { ... package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  29. 58.

    for _, decl := range node.Decls { // search for

    main function fn, ok := decl.(*ast.FuncDecl) if !ok { continue } // inside the main function body for _, stmt := range fn.Body.List { // search through statements for a declaration... declStmt, ok := stmt.(*ast.DeclStmt) if !ok { continue } // continue with declStmt.Decl // ... } } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  30. 59.

    for _, decl := range node.Decls { // search for

    main function fn, ok := decl.(*ast.FuncDecl) if !ok { continue } // inside the main function body for _, stmt := range fn.Body.List { // search through statements for a declaration... declStmt, ok := stmt.(*ast.DeclStmt) if !ok { continue } // continue with declStmt.Decl // ... } } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  31. 60.

    // continue with declStmt.Decl genDecl, ok := declStmt.Decl.(*ast.GenDecl) if !ok

    { continue } // declarations can have multiple specs, // search for a valuespec for _, spec := range genDecl.Specs { valueSpec, ok := spec.(*ast.ValueSpec) if !ok { continue } // continue with valueSpec.Values // ... } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  32. 61.

    // continue with declStmt.Decl genDecl, ok := declStmt.Decl.(*ast.GenDecl) if !ok

    { continue } // declarations can have multiple specs, // search for a valuespec for _, spec := range genDecl.Specs { valueSpec, ok := spec.(*ast.ValueSpec) if !ok { continue } // continue with valueSpec.Values // ... } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  33. 62.

    // continue with valueSpec.Values for _, expr := range valueSpec.Values

    { // search for a binary expr binaryExpr, ok := expr.(*ast.BinaryExpr) if !ok { continue } // found it! fmt.Printf("Found binary expression at: %d:%d\n", fset.Position(binaryExpr.Pos()).Line, fset.Position(binaryExpr.Pos()).Column, ) } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  34. 63.

    // continue with valueSpec.Values for _, expr := range valueSpec.Values

    { // search for a binary expr binaryExpr, ok := expr.(*ast.BinaryExpr) if !ok { continue } // found it! fmt.Printf("Found binary expression at: %d:%d\n", fset.Position(binaryExpr.Pos()).Line, fset.Position(binaryExpr.Pos()).Column, ) } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } 1 2 3 4 5 6 7 8
  35. 64.

    // continue with valueSpec.Values for _, expr := range valueSpec.Values

    { // search for a binary expr binaryExpr, ok := expr.(*ast.BinaryExpr) if !ok { continue } // found it! fmt.Printf("Found binary expression at: %d:%d\n", fset.Position(binaryExpr.Pos()).Line, fset.Position(binaryExpr.Pos()).Column, ) } package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } Output: Found binary expression at: 6:17 1 2 3 4 5 6 7 8
  36. 69.

    visitorFunc := func(n ast.Node) bool { binaryExpr, ok := n.(*ast.BinaryExpr)

    if !ok { return true } fmt.Printf("Found binary expression at: %d:%d\n", fset.Position(binaryExpr.Pos()).Line, fset.Position(binaryExpr.Pos()).Column, ) return true } // walk trough all nodes and run the // given function for each node ast.Inspect(node, visitorFunc)
  37. 70.

    visitorFunc := func(n ast.Node) bool { binaryExpr, ok := n.(*ast.BinaryExpr)

    if !ok { return true } fmt.Printf("Found binary expression at: %d:%d\n", fset.Position(binaryExpr.Pos()).Line, fset.Position(binaryExpr.Pos()).Column, ) return true } // walk trough all nodes and run the // given function for each node ast.Inspect(node, visitorFunc) Output: Found binary expression at: 6:17
  38. 73.
  39. 75.

    func main() { src := []byte(`package main var sum int

    = 3 + 2`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } printer.Fprint(os.Stdout, fset, node) }
  40. 76.

    func main() { src := []byte(`package main var sum int

    = 3 + 2`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } printer.Fprint(os.Stdout, fset, node) }
  41. 77.

    func main() { src := []byte(`package main var sum int

    = 3 + 2`) fset := token.NewFileSet() node, err := parser.ParseFile(fset, "demo", src, parser.ParseComments) if err != nil { panic(err) } printer.Fprint(os.Stdout, fset, node) } Output: package main var sum int = 3 + 2
  42. 79.

    Basic rules for a simple tool 1. Read (go/parser) 2.

    Inspect (go/ast) 3. Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  43. 80.

    Case study: motion 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  44. 81.

    func NewParser(opts *ParserOptions) (*Parser, error) { // ... switch {

    case opts.File != "": p.file, err = parser.ParseFile(fset, opts.File, nil, mode) case opts.Dir != "": p.pkgs, err = parser.ParseDir(fset, opts.Dir, nil, mode) case opts.Src != nil: p.file, err = parser.ParseFile(fset, "src.go", opts.Src, mode) default: return nil, errors.New("file, src or dir is not specified") } return p, nil }
  45. 82.

    func NewParser(opts *ParserOptions) (*Parser, error) { // ... switch {

    case opts.File != "": p.file, err = parser.ParseFile(fset, opts.File, nil, mode) case opts.Dir != "": p.pkgs, err = parser.ParseDir(fset, opts.Dir, nil, mode) case opts.Src != nil: p.file, err = parser.ParseFile(fset, "src.go", opts.Src, mode) default: return nil, errors.New("file, src or dir is not specified") } return p, nil }
  46. 83.

    func NewParser(opts *ParserOptions) (*Parser, error) { // ... switch {

    case opts.File != "": p.file, err = parser.ParseFile(fset, opts.File, nil, mode) case opts.Dir != "": p.pkgs, err = parser.ParseDir(fset, opts.Dir, nil, mode) case opts.Src != nil: p.file, err = parser.ParseFile(fset, "src.go", opts.Src, mode) default: return nil, errors.New("file, src or dir is not specified") } return p, nil }
  47. 84.

    func NewParser(opts *ParserOptions) (*Parser, error) { // ... switch {

    case opts.File != "": p.file, err = parser.ParseFile(fset, opts.File, nil, mode) case opts.Dir != "": p.pkgs, err = parser.ParseDir(fset, opts.Dir, nil, mode) case opts.Src != nil: p.file, err = parser.ParseFile(fset, "src.go", opts.Src, mode) default: return nil, errors.New("file, src or dir is not specified") } return p, nil }
  48. 85.

    Case study: motion 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  49. 86.

    func (p *Parser) collectFunctions() []ast.Node { var funcs []*ast.FuncDecl visitorFunc

    := func(n ast.Node) bool { fn, ok := n.(*ast.FuncDecl) if !ok { return true } funcs = append(funcs, fn) return true } ast.Inspect(p.file, visitorFunc) return funcs }
  50. 87.

    func (p *Parser) collectFunctions() []ast.Node { var funcs []*ast.FuncDecl visitorFunc

    := func(n ast.Node) bool { fn, ok := n.(*ast.FuncDecl) if !ok { return true } funcs = append(funcs, fn) return true } ast.Inspect(p.file, visitorFunc) return funcs }
  51. 88.

    func (p *Parser) collectFunctions() []ast.Node { var funcs []*ast.FuncDecl visitorFunc

    := func(n ast.Node) bool { fn, ok := n.(*ast.FuncDecl) if !ok { return true } funcs = append(funcs, fn) return true } ast.Inspect(p.file, visitorFunc) return funcs }
  52. 89.

    func (p *Parser) collectFunctions() []ast.Node { var funcs []*ast.FuncDecl visitorFunc

    := func(n ast.Node) bool { fn, ok := n.(*ast.FuncDecl) if !ok { return true } funcs = append(funcs, fn) return true } ast.Inspect(p.file, visitorFunc) return funcs }
  53. 90.

    func (p *Parser) collectFunctions() []ast.Node { var funcs []*ast.FuncDecl visitorFunc

    := func(n ast.Node) bool { fn, ok := n.(*ast.FuncDecl) if !ok { return true } funcs = append(funcs, fn) return true } ast.Inspect(p.file, visitorFunc) return funcs }
  54. 91.

    Case study: motion 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  55. 92.

    func formatFunctions(funcs []*ast.FuncDecl) string { out := new(bytes.Buffer) for _,

    fn := range funcs { fnPos := p.fset.Position(fn.Type.Func) // foo.go:line:column filePos := fmt.Sprintf("%s:%d:%d", fnPos.Filename, fnPos.Line, fnPos.Column) // foo.go:line:column | func name | func signature() fmt.Fprintf(out, "%s | %s | %s\n", filePos, fn.Name.Name, funcSignature(fn)) } return out.String() }
  56. 93.

    func formatFunctions(funcs []*ast.FuncDecl) string { out := new(bytes.Buffer) for _,

    fn := range funcs { fnPos := p.fset.Position(fn.Type.Func) // foo.go:line:column filePos := fmt.Sprintf("%s:%d:%d", fnPos.Filename, fnPos.Line, fnPos.Column) // foo.go:line:column | func name | func signature() fmt.Fprintf(out, "%s | %s | %s\n", filePos, fn.Name.Name, funcSignature(fn)) } return out.String() }
  57. 94.

    func formatFunctions(funcs []*ast.FuncDecl) string { out := new(bytes.Buffer) for _,

    fn := range funcs { fnPos := p.fset.Position(fn.Type.Func) // foo.go:line:column filePos := fmt.Sprintf("%s:%d:%d", fnPos.Filename, fnPos.Line, fnPos.Column) // foo.go:line:column | func name | func signature() fmt.Fprintf(out, "%s | %s | %s\n", filePos, fn.Name.Name, funcSignature(fn)) } return out.String() }
  58. 95.

    func formatFunctions(funcs []*ast.FuncDecl) string { out := new(bytes.Buffer) for _,

    fn := range funcs { fnPos := p.fset.Position(fn.Type.Func) // foo.go:line:column filePos := fmt.Sprintf("%s:%d:%d", fnPos.Filename, fnPos.Line, fnPos.Column) // foo.go:line:column | func name | func signature() fmt.Fprintf(out, "%s | %s | %s\n", filePos, fn.Name.Name, funcSignature(fn)) } return out.String() }
  59. 96.

    Case study: motion 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  60. 97.

    func main() { flagFile = flag.String("file", "", "Filename to be

    parsed") flagMode = flag.String("mode", "", "Running mode") flag.Parse() opts := &ParserOptions{File: *flagFile} parser, err := NewParser(opts) if err != nil { panic(err) } if *flagMode == "funcs" { funcs := parser.collectFunctions() out := formatFunctions(funcs) fmt.Println(out) } // ... }
  61. 98.

    func main() { flagFile = flag.String("file", "", "Filename to be

    parsed") flagMode = flag.String("mode", "", "Running mode") flag.Parse() opts := &ParserOptions{File: *flagFile} parser, err := NewParser(opts) if err != nil { panic(err) } if *flagMode == "funcs" { funcs := parser.collectFunctions() out := formatFunctions(funcs) fmt.Println(out) } // ... }
  62. 99.

    func main() { flagFile = flag.String("file", "", "Filename to be

    parsed") flagMode = flag.String("mode", "", "Running mode") flag.Parse() opts := &ParserOptions{File: *flagFile} parser, err := NewParser(opts) if err != nil { panic(err) } if *flagMode == "funcs" { funcs := parser.collectFunctions() out := formatFunctions(funcs) fmt.Println(out) } // ... }
  63. 100.

    func main() { flagFile = flag.String("file", "", "Filename to be

    parsed") flagMode = flag.String("mode", "", "Running mode") flag.Parse() opts := &ParserOptions{File: *flagFile} parser, err := NewParser(opts) if err != nil { panic(err) } if *flagMode == "funcs" { funcs := parser.collectFunctions() out := formatFunctions(funcs) fmt.Println(out) } // ... }
  64. 101.

    $ cat main.go package main import "fmt" func main() {

    a := 5 incr := func(x int) int { fmt.Printf("incremeting %d by one\n", x) return x + 1 } b := incr(a) fmt.Println(b, check(b)) } // check checks if the given integer // is smaller than 10 func check(a int) bool { return a < 10 } // decr decrements the given integer func decr(x int) int { return x - 1 }
  65. 102.

    $ motion -mode funcs -file main.go main.go:5:1 | main |

    func main() main.go:19:1 | check | func check(a int) bool main.go:24:1 | decr | func decr(x int) int
  66. 103.
  67. 104.
  68. 106.

    Case study: gofmt 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  69. 107.
  70. 108.
  71. 109.

    Case study: gofmt go/parser go/ast package main import "fmt" func

    main(){ var sum int = 3+2 fmt.Println( sum ) }
  72. 110.

    func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error

    { // ... src, err := ioutil.ReadAll(in) if err != nil { return err } file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, stdin) if err != nil { return err } // ...
  73. 111.

    func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error

    { // ... src, err := ioutil.ReadAll(in) if err != nil { return err } file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, stdin) if err != nil { return err } // ...
  74. 112.

    Case study: gofmt 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  75. 113.

    Case study: gofmt go/parser go/ast sort simplify rewrite package main

    import "fmt" func main(){ var sum int = 3+2 fmt.Println( sum ) }
  76. 114.

    func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error

    { // ... if rewrite != nil { if sourceAdj == nil { file = rewrite(file) } else { fmt.Fprintf(os.Stderr, "warning: rewrite ignored for incomplete programs\n") } } ast.SortImports(fileSet, file) if *simplifyAST { simplify(file) } // ...
  77. 115.

    Case study: gofmt 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  78. 116.

    Case study: gofmt go/parser go/ast sort simplify rewrite go/printer package

    main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } package main import "fmt" func main(){ var sum int = 3+2 fmt.Println( sum ) }
  79. 117.

    func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error

    { // ... res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth}) if err != nil { return err } if !*list && !*write && !*doDiff { _, err = out.Write(res) } // ...
  80. 118.

    Case study: gofmt 1. Read (go/parser) 2. Inspect (go/ast) 3.

    Write (go/printer) or custom format (JSON, etc...) 4. Create a CLI
  81. 119.

    func main() { // ... for i := 0; i

    < flag.NArg(); i++ { path := flag.Arg(i) switch dir, err := os.Stat(path); { case err != nil: report(err) case dir.IsDir(): walkDir(path) default: if err := processFile(path, nil, os.Stdout, false); err != nil { report(err) } } } // ... }
  82. 120.

    $ cat main.go package main import "fmt" func main() {

    var sum int = 3+2 fmt.Println( sum ) }
  83. 121.

    $ cat main.go package main import "fmt" func main() {

    var sum int = 3+2 fmt.Println( sum ) } $ gofmt main.go package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) }
  84. 122.
  85. 123.

    package main import "fmt" func main() { var sum int

    = 3 + 2 fmt.Println(sum) } go/parser go/ast go/printer package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) }
  86. 124.

    package main import "fmt" func main() { var sum int

    = 3 + 2 fmt.Println(sum) } go/ast package main import "fmt" func main() { var sum int = 3 + 2 fmt.Println(sum) } go/printer go/parser go/scanner go/token go/parser
  87. 125.