RequestGen

Request builder generator with Go generate.

Building Request Builder without Pain.

Transcript

1. c9s @ 2022 Mar 27
   requestgen
   Request builder generator with Go generate
   

   View Slide

2. What’s request builder?
   • A request builder helps you build up the HTTP request object for the API.
   
   • A request builder contains a set of methods that help you set the parameters.
   

   View Slide

3. What’s request builder generator
   • A request builder generator helps you generate the request builder
   
   • With a simple schema or con
   fi
   guration.
   
   • DRY (Do not repeat yourself)
   

   View Slide

4. Why write a request builder generator
   • Di
   ff
   erent Authentication Methods (Some are very complicated)
   
   • Response structure could be di
   ff
   erent.
   
   • Some exchanges return paginated results.
   

   View Slide

5. The Request Builder API
   The Request Build Usage From adshao/go-binance
   

   View Slide

6. The hand-written Implementation
   

   View Slide

7. We Want Some Automation
   

   View Slide

8. The Same API
   

   View Slide

9. But Generate It Automatically
   

   View Slide

10. How to use it?
    

    View Slide

11. Basic Usage
    

    View Slide

12. type PlaceOrderRequest struct {
    client requestgen.APIClient
    symbol string `param:"symbol,required"`
    }
    Your APIClient implementation for
    sending requests
    

    View Slide

13. type PlaceOrderRequest struct {
    client requestgen.APIClient
    symbol string `param:"symbol,required"`
    }
    symbol=BTCUSDT
    
    
    In the request body
    

    View Slide

14. //go:generate requestgen —type PlaceOrderRequest
    type PlaceOrderRequest struct {
    client requestgen.APIClient
    symbol string `param:"symbol,required"`
    }
    Add the go generate command with your
    type name
    

    View Slide

15. //go:generate requestgen —type PlaceOrderRequest
    type PlaceOrderRequest struct {
    client requestgen.APIClient
    symbol string `param:"symbol,required"`
    }
    Equivalent to this command
    curl -X POST -F symbol=BTCUSDT https://…….
    

    View Slide

16. // APIClient defines the request builder method and request method for the API service
    type APIClient interface {
    // NewRequest builds up the http request for public endpoints
    NewRequest(ctx context.Context, method, refURL string, params url.Values, payload interface{}) (*http.Request, error)
    // SendRequest sends the request object to the api gateway
    SendRequest(req *http.Request) (*Response, error)
    }
    Simply implement the following interface to support HTTP request:
    

    View Slide

17. ctx := context.Background()
    client := NewClient()
    Allocate your HTTP client
    

    View Slide

18. ctx := context.Background()
    client := NewClient()
    req := PlaceOrderRequest{client: client}
    Construct your request builder
    

    View Slide

19. ctx := context.Background()
    client := NewClient()
    req := PlaceOrderRequest{client: client}
    req.Symbol(“BTCUSDT").
    ClientOrderID(“0aa1a0123").
    Side(SideTypeBuy).
    OrdType(OrderTypeLimit)
    Assign values
    

    View Slide

20. ctx := context.Background()
    client := NewClient()
    req := PlaceOrderRequest{client: client}
    req.Symbol(“BTCUSDT").
    ClientOrderID(“0aa1a0123").
    Side(SideTypeBuy).
    OrdType(OrderTypeLimit)
    response, err := req.Do(ctx)
    if err != nil {
    // ...
    } Send request
    

    View Slide

21. Adding More Fields
    

    View Slide

22. type PlaceOrderRequest struct {
    client requestgen.APIClient
    clientOrderID *string `param:"clientOid,required" defaultValuer:"uuid()"`
    }
    Default value UUID
    

    View Slide

23. * Pointer
    fi
    eld = optional
    fi
    eld
    type PlaceOrderRequest struct {
    client requestgen.APIClient
    tag *string `param:"tag"`
    }
    

    View Slide

24. type PlaceOrderRequest struct {
    client requestgen.APIClient
    // "buy" or "sell"
    side SideType `param:"side,required" validValues:"buy,sell"`
    }
    Simple Value Validator
    
    
    For String type and Integer type
    

    View Slide

25. type PlaceOrderRequest struct {
    client requestgen.APIClient
    // "buy" or "sell"
    side SideType `param:"side,required" validValues:"buy,sell"`
    }
    User-de
    fi
    ned types are supported
    type SideType string
    const (
    SideTypeBuy SideType = "buy"
    SideTypeSell SideType = "sell"
    )
    

    View Slide

26. type PlaceOrderRequest struct {
    client requestgen.APIClient
    startTime *time.Time `param:"startTime,milliseconds" defaultValuer:"now()"`
    }
    Millisecond timestamp conversion
    curl -X POST -F startTime=1648467987762 https://…….
    

    View Slide

27. type PlaceOrderRequest struct {
    client requestgen.APIClient
    // page defines the query parameters for something like '?page=123'
    page *int64 `param:"page,query"`
    }
    Query parameter in the POST request
    

    View Slide

28. Parsing Response
    

    View Slide

29. {
    “code”: 0,
    “data”: { … }
    “page”: 1
    }
    Response Data Container
    

    View Slide

30. //go:generate requestgen
    -type PlaceOrderRequest
    -responseType .Response
    -responseDataField Data
    -responseDataType .Order
    type Response struct {
    Code string `json:"code"`
    Message string `json:"msg"`
    CurrentPage int `json:"currentPage"`
    PageSize int `json:"pageSize"`
    TotalNum int `json:"totalNum"`
    TotalPage int `json:"totalPage"`
    Data json.RawMessage `json:"data"`
    }
    Response Type
    

    View Slide

31. //go:generate requestgen
    -type PlaceOrderRequest
    -responseType .Response
    -responseDataField Data
    -responseDataType .Order
    type Response struct {
    Code string `json:"code"`
    Message string `json:"msg"`
    CurrentPage int `json:"currentPage"`
    PageSize int `json:"pageSize"`
    TotalNum int `json:"totalNum"`
    TotalPage int `json:"totalPage"`
    Data json.RawMessage `json:"data"`
    }
    Response Data Field
    Response Data Field
    

    View Slide

32. //go:generate requestgen
    -type PlaceOrderRequest
    -responseType .Response
    -responseDataField Data
    -responseDataType .Order
    type Response struct {
    Code string `json:"code"`
    Message string `json:"msg"`
    CurrentPage int `json:"currentPage"`
    PageSize int `json:"pageSize"`
    TotalNum int `json:"totalNum"`
    TotalPage int `json:"totalPage"`
    Data json.RawMessage `json:"data"`
    }
    Parsed into the Order struct
    

    View Slide

33. var apiResponse APIResponse
    if err := response.DecodeJSON(&apiResponse); err != nil {
    return nil, err
    }
    var data OrderResponse
    if err := json.Unmarshal(apiResponse.Data, &data); err != nil {
    return nil, err
    }
    return &data, nil
    The generated code
    Response Wrapper Type
    

    View Slide

34. var apiResponse APIResponse
    if err := response.DecodeJSON(&apiResponse); err != nil {
    return nil, err
    }
    var data OrderResponse
    if err := json.Unmarshal(apiResponse.Data, &data); err != nil {
    return nil, err
    }
    return &data, nil
    The generated code
    Data Type
    

    View Slide

35. var apiResponse APIResponse
    if err := response.DecodeJSON(&apiResponse); err != nil {
    return nil, err
    }
    var data OrderResponse
    if err := json.Unmarshal(apiResponse.Data, &data); err != nil {
    return nil, err
    }
    return &data, nil
    The generated code
    Data Field
    

    View Slide

36. Request Methods
    

    View Slide

37. go:generate requestgen -method POST -type PlaceOrderRequest
    go:generate requestgen -method GET -type PlaceOrderRequest
    go:generate requestgen -method PUT -type PlaceOrderRequest
    go:generate requestgen -method DELETE -type PlaceOrderRequest
    HTTP Method
    

    View Slide

38. //go:generate -command GetRequest requestgen -method GET
    //go:generate -command PostRequest requestgen -method POST
    //go:generate GetRequest -type ListSymbolsRequest -url "/api/v1/symbols"
    -responseDataType []Symbol
    Go Generate Alias
    

    View Slide

39. Return Type
    

    View Slide

40. //go:generate GetRequest
    -type ListSymbolsRequest
    -url "/api/v1/symbols" -responseDataType []Symbol
    Return Slice Data Type
    

    View Slide

41. Return Simple Struct Type
    //go:generate GetRequest
    -type GetAllTickersRequest
    -url "/api/v1/market/allTickers"
    -responseDataType AllTickers
    

    View Slide

42. How Does It Work?
    

    View Slide

43. Go 1.5
    transit to Go compiler from C compiler
    

    View Slide

44. go/types
    Go’s type checker
    

    View Slide

45. Parsing
    
    
    go/{parser, ast, scanner, token}
    Type Checker
    
    
    go/types
    vet lint eg gorename stringer mockgen mockery
    

    View Slide

46. a.go
    c.go
    b.go
    fmt
    Type Checker
    
    
    go/types
    compilation errors
    expressions
    type-checked packages
    imported
    
    
    packages
    

    View Slide

47. Step 1
    parse package
    fi
    les
    package main
    func parsePackage(patterns []string, tags []string) {
    cfg := &packages.Config{
    Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles |
    packages.NeedImports |
    packages.NeedTypes | packages.NeedTypesSizes |
    packages.NeedTypesInfo | packages.NeedFiles |
    packages.NeedSyntax | packages.NeedTypesInfo,
    Tests: false,
    BuildFlags: []string{fmt.Sprintf("-tags=%s", strings.Join(tags, " "))},
    }
    pkgs, err := packages.Load(cfg, patterns...)
    if err != nil {
    log.Fatal(err)
    }
    }
    

    View Slide

48. Step 2
    traverse the syntax tree
    for _, pkg := range pkgs {
    for i, file := range pkg.Syntax {
    ast.Inspect(file.file, func(node ast.Node) bool {
    switch decl := node.(type) {
    case *ast.ImportSpec:
    case *ast.FuncDecl:
    // ....
    }
    return true
    }
    }
    }
    USVFJUFSBUFJOUPUIFOPEF CMPDL TUBUFNFOU GVODUJPOʜFUD
    
    UIFOPEFUZQFUIBUZPVXBOUUPJOTQFDU
    

    View Slide

49. type (
    // An ImportSpec node represents a single package import.
    ImportSpec struct {
    Doc *CommentGroup // associated documentation; or nil
    Name *Ident // local package name (including "."); or nil
    Path *BasicLit // import path
    Comment *CommentGroup // line comments; or nil
    EndPos token.Pos // end of spec (overrides Path.Pos if nonzero)
    }
    // A ValueSpec node represents a constant or variable declaration
    // (ConstSpec or VarSpec production).
    //
    ValueSpec struct {
    Doc *CommentGroup // associated documentation; or nil
    Names []*Ident // value names (len(Names) > 0)
    Type Expr // value type; or nil
    Values []Expr // initial values; or nil
    Comment *CommentGroup // line comments; or nil
    }
    )
    import (
    "golang.org/x/tools/go/packages"
    )
    a := 1 + 2
    

    View Slide

50. Step 3
    Resolve the AST node to Type Info
    switch decl := node.(type) {
    case *ast.FuncDecl:
    // get the first receiver node
    receiver := decl.Recv.List[0]
    // pass the receiver type token to the type info map
    receiverTypeAndValue, ok := pkg.TypesInfo.Types[receiver.Type]
    if !ok {
    return true
    }
    // inspect the REAL type
    switch v := recvTV.Type.(type) {
    case *types.Named: // type A int
    case *types.Pointer: // *int
    }
    }
    5ZQF"OE7BMVFDPOUBJOTUIFUZQFJOGPSNBUJPO
    VTFUIFUZQFJOGPSNBUJPOUPDIFDL
    

    View Slide

51. Step 4
    generate the code
    func renderCode() {
    var sliceCallbackTmpl = template.New("slice-callbacks").Funcs(funcMap)
    sliceCallbackTmpl = template.Must(sliceCallbackTmpl.Parse(`
    func ( {{- .RecvName }} *{{ .Field.StructName -}} ) On{{- .Field.EventName -}} (cb {{ .Field.CallbackTypeName .Qualifier -}} )
    {{ .RecvName }}.{{ .Field.FieldName }} = append({{- .RecvName }}.{{ .Field.FieldName }}, cb)
    }
    func ( {{- .RecvName }} *{{ .Field.StructName -}} ) Emit{{- .Field.EventName -}} {{ .Field.CallbackParamsTuple | typeString }}
    for _, cb := range {{ .RecvName }}.{{ .Field.FieldName }} {
    cb({{ .Field.CallbackParamsVarNames | join ", " }})
    }
    }
    `)
    sliceCallbackTmpl.Execute(&buf, TemplateArgs{
    Field: field,
    RecvName: recvName,
    Qualifier: qf,
    GenerateRemoveMethod: *generateRemoveMethod,
    })
    }
    

    View Slide

52. Step 5
    reformat the code
    import (
    "go/format"
    )
    src, err := format.Source(buf.Bytes())
    if err != nil {
    // Should never happen, but can arise when developing this code.
    // The user can compile the output to see the error.
    log.Printf("warning: internal error: invalid Go generated: %s", err)
    log.Printf("warning: compile the package to analyze the error")
    return buf.Bytes()
    }
    return src
    

    View Slide

53. Step 6
    compile and test your code!
    

    View Slide

54. Q & A
    

    View Slide