Demystifying DSLs for better analysis and understanding

Presented at RubyKaigi 2021

The ability to create DSLs is one of the biggest strengths of Ruby. They allow us to write easy to use interfaces and reduce the need for boilerplate code. On the flip side, DSLs encapsulate complex logic which makes it hard for developers to understand what’s happening under the covers.

Surfacing DSLs as static artifacts makes working with them much easier. Generating RBI/RBS files that declare the methods which are dynamically created at runtime, allows static analyzers like Sorbet or Steep to work with DSLs. This also allows for better developer tooling.

Transcript

1. $
   

   View Slide

2. rails new myapp
   $
   

   View Slide

3. create
   create README.md
   create Rakefile
   create .ruby-version
   create config.ru
   create .gitignore
   create .gitattributes
   create Gemfile
   run git init from "."
   ...
   create tmp
   $
   rails new myapp
   $
   

   View Slide

4. (myapp) $
   

   View Slide

5. bin/rails generate model User name:string role:string
   (myapp) $
   

   View Slide

6. invoke active_record
   create db/migrate/20120909070000_create_users.rb
   create app/models/user.rb
   (myapp) $
   bin/rails generate model User name:string role:string
   (myapp) $
   

   View Slide

7. invoke active_record
   create db/migrate/20120909070000_create_users.rb
   create app/models/user.rb
   (myapp) $ bat app/models/user.rb
   bin/rails generate model User name:string role:string
   (myapp) $
   

   View Slide

8. invoke active_record
   create db/migrate/20120909070000_create_users.rb
   create app/models/user.rb
   (myapp) $ bat app/models/user.rb
   bin/rails generate model User name:string role:string
   (myapp) $
   ───────┬──────────────────────────────────────────────────────────────────────
   │ File: app/models/user.rb
   ───────┼──────────────────────────────────────────────────────────────────────
   1 │ class User < ApplicationRecord
   2 │ end
   ───────┴──────────────────────────────────────────────────────────────────────
   

   View Slide

9. invoke active_record
   create db/migrate/20120909070000_create_users.rb
   create app/models/user.rb
   (myapp) $ bat app/models/user.rb
   bin/rails generate model User name:string role:string
   (myapp) $
   ───────┬──────────────────────────────────────────────────────────────────────
   │ File: app/models/user.rb
   ───────┼──────────────────────────────────────────────────────────────────────
   1 │ class User < ApplicationRecord
   2 │ end
   ───────┴──────────────────────────────────────────────────────────────────────
   🤔
   

   View Slide

10. Demystifying DSLs
    
    
    for better analysis and understanding
    Ufuk Kayserilioglu
    
    @paracycle
    

    View Slide

11. What is a DSL anyway?
    

    View Slide

12. Domain Speci
    fi
    c Language
    

    View Slide

13. Uses Ruby metaprogramming
    

    View Slide

14. Gemfile
    

    View Slide

15. Rakefile
    

    View Slide

16. RSpec tests
    

    View Slide

17. Rails!
    

    View Slide

18. So what's so cool about them?
    

    View Slide

19. No boilerplate
    

    View Slide

20. Natural API
    

    View Slide

21. Great, how can I build one?
    

    View Slide

22. class CreditCard
    
    
    include Encryptable
    
    
    attr_encrypted :number
    
    
    end
    

    View Slide

23. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    

    View Slide

24. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    

    View Slide

25. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    

    View Slide

26. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    

    View Slide

27. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    

    View Slide

28. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    

    View Slide

29. module Encryptable
    
    
    def self.included(mod)
    
    
    mod.extend(ClassMethods)
    
    
    end
    
    
    module ClassMethods
    
    
    def attr_encrypted(attr_name)
    
    
    attr_accessor(attr_name)
    
    
    encrypted_attr_name = "#{attr_name}_encrypted"
    
    
    define_method(encrypted_attr_name) do
    
    
    value = send(attr_name)
    
    
    encrypt(value)
    
    
    end
    
    
    define_method("#{encrypted_attr_name}=") do |value|
    
    
    send("#{attr_name}=", decrypt(value))
    
    
    end
    
    
    end
    
    
    end
    
    
    private
    
    
    def encrypt(value)
    
    
    value.unpack("H*").first
    
    
    end
    
    
    def decrypt(value)
    
    
    [value].pack("H*")
    
    
    end
    
    
    end
    card = CreditCard.new
    
    
    card.number = "1234 5678 9012 3456"
    
    
    p card.number
    
    
    #
    = >
    "1234 5678 9012 3456"
    
    
    p card.number_encrypted
    
    
    #
    = >
    "31323334203536373820393031322033343536"
    
    
    card.number_encrypted = "33343536"
    
    
    p card.number
    
    
    #
    = >
    "3456"
    

    View Slide

30. So, it's all sunshine and roses?
    

    View Slide

31. Static Analysis
    

    View Slide

32. Code Readability
    

    View Slide

33. # message.rb
    
    
    require "smart_properties"
    
    
    class Message
    
    
    include SmartProperties
    
    
    property! :subject, accepts: String, default: "(No Subject)"
    
    
    property :body, accepts: String
    
    
    property :time, accepts: Time, default:
    - >
    { Time.now }
    
    
    end
    
    
    message = Message.new(subject: "New Message", body: "Lorem ipsum dolor sit amet")
    
    
    puts "Message '#{message.subject}' created at #{message.time} has body: #{message.body?}"
    

    View Slide

34. # message.rb
    
    
    require "smart_properties"
    
    
    class Message
    
    
    include SmartProperties
    
    
    property! :subject, accepts: String, default: "(No Subject)"
    
    
    property :body, accepts: String
    
    
    property :time, accepts: Time, default:
    - >
    { Time.now }
    
    
    end
    
    
    message = Message.new(subject: "New Message", body: "Lorem ipsum dolor sit amet")
    
    
    puts "Message '#{message.subject}' created at #{message.time} has body: #{message.body?}"
    $ ruby message.rb
    
    
    message.rb:15:in `': undefined method `body?' for #Message", @time=2021-09-09 07:00:00Z, @body="Lorem ipsum dolor sit amet"> (NoMethodError)
    
    
    Did you mean? body
    
    
    body=
    

    View Slide

35. RBI / RBS
    

    View Slide

36. # message.rbi
    
    
    # typed: true
    
    
    class Message
    
    
    def subject; end
    
    
    def subject=(value); end
    
    
    def body; end
    
    
    def body=(value); end
    
    
    def time; end
    
    
    def time=(value); end
    
    
    end
    

    View Slide

37. $ bundle exec srb tc
    
    
    message.rb:15: Method body? does not exist on Message https://srb.help/7003
    
    
    15 |puts "Message '#{message.subject}' created at #{message.time} has body: #{message.body?}"
    
    
    ^^^^^^^^^^^^^
    # message.rbi
    
    
    # typed: true
    
    
    class Message
    
    
    def subject; end
    
    
    def subject=(value); end
    
    
    def body; end
    
    
    def body=(value); end
    
    
    def time; end
    
    
    def time=(value); end
    
    
    end
    

    View Slide

38. Do I have to write RBI
    fi
    les?
    

    View Slide

39. Tapioca
    

    View Slide

40. class Tapioca
    :
    :
    Compilers
    ::
    Dsl
    ::
    Base
    
    
    extend T
    ::
    Sig
    
    
    extend T
    ::
    Helpers
    
    
    abstract!
    
    
    sig { abstract.params(tree: RBI
    : :
    Tree, constant: T.untyped).void }
    
    
    def decorate(tree, constant); end
    
    
    sig { abstract.returns(T
    ::
    Enumerable[Module]) }
    
    
    def gather_constants; end
    
    
    # ...
    
    
    end
    

    View Slide

41. class SmartPropertiesGenerator < Tapioca
    ::
    Compilers
    : :
    Dsl
    : :
    Base
    
    
    def decorate(root, smart_prop)
    
    
    properties = smart_prop.properties
    
    
    return if properties.keys.empty?
    
    
    root.create_path(smart_prop) do |klass|
    
    
    properties.values.each do |property|
    
    
    name = property.name.to_s
    
    
    klass.create_method(name)
    
    
    klass.create_method("#{name}=", parameters: [create_param("value", type: "T.untyped")])
    
    
    end
    
    
    end
    
    
    end
    
    
    def gather_constants
    
    
    ObjectSpace.each_object(Class).select { |c| c <
    ::
    SmartProperties }
    
    
    end
    
    
    end
    
    
    

    View Slide

42. # message.rbi
    
    
    # typed: true
    
    
    class Message
    
    
    sig { returns("T.untyped") }
    
    
    def subject; end
    
    
    sig { params(value: "T.untyped").returns("T.untyped") }
    
    
    def subject=(value); end
    
    
    sig { returns("T.untyped") }
    
    
    def body; end
    
    
    sig { params(value: "T.untyped").returns("T.untyped") }
    
    
    def body=(value); end
    
    
    sig { returns("T.untyped") }
    
    
    def time; end
    
    
    sig { params(value: "T.untyped").returns("T.untyped") }
    
    
    def time=(value); end
    
    
    end
    
    
    

    View Slide

43. View Slide

44. RBS Rails
    

    View Slide

45. Why should I care?
    

    View Slide

46. Static Analysis
    Code Understanding
    㱺
    

    View Slide

47. $ bin/tapioca dsl User
    
    
    Loading Rails application... Done
    
    
    Loading DSL generator classes... Done
    
    
    Compiling DSL RBI files...
    
    
    Wrote: sorbet/rbi/dsl/user.rbi
    
    
    Done
    
    
    All operations performed in working directory.
    
    
    Please review changes and commit them.
    

    View Slide

48. # DO NOT EDIT MANUALLY
    
    
    # This is an autogenerated file for dynamic methods in `User`.
    
    
    # Please instead update this file by running `bin/tapioca dsl User`.
    
    
    # typed: true
    
    
    class User
    
    
    include GeneratedAttributeMethods
    
    
    module GeneratedAttributeMethods
    
    
    sig { returns(T.nilable(
    :
    :
    ActiveSupport
    ::
    TimeWithZone)) }
    
    
    def created_at; end
    
    
    sig { params(value:
    ::
    ActiveSupport
    ::
    TimeWithZone).returns(
    ::
    ActiveSupport
    ::
    TimeWithZone) }
    
    
    def created_at=(value); end
    
    
    # ...
    
    
    sig { returns(T.nilable(
    :
    :
    String)) }
    
    
    def name; end
    
    
    sig { params(value: T.nilable(
    ::
    String)).returns(T.nilable(
    ::
    String)) }
    
    
    def name=(value); end
    
    
    # ...
    
    
    sig { returns(T.nilable(
    :
    :
    String)) }
    
    
    def role; end
    
    
    sig { params(value: T.nilable(
    ::
    String)).returns(T.nilable(
    ::
    String)) }
    
    
    def role=(value); end
    
    
    # ...
    
    
    end
    
    
    

    View Slide

49. View Slide

50. View Slide

51. That's neat, actually!
    

    View Slide

52. Thank you!
    The Shopify After-hours AMA
    
    Friday 3pm JST
    
    https://bit.ly/shopifyamaregister
    
    

    View Slide