SnowOne 2020: Jabel – retrofitting Java Compiler by instrumenting it!

Transcript

1. Jabel – retrofitting Java
   Compiler by instrumenting it!
   Sergei Egorov
   @bsideup
   

   View Slide

2. About me
   • Staff Engineer at Pivotal’s Spring R&D, working on Project Reactor ⚛
   
   • Java Champion
   
   • Testcontainers co-maintainer
   
   • Berlin Spring User Group co-organizer
   
   • Developer tools geek
   @bsideup
   

   View Slide

3. How compilers work?
   

   View Slide

4. Very easy!
   @bsideup
   

   View Slide

5. Source
   code
   @bsideup
   

   View Slide

6. Source
   code
   Scanner
   @bsideup
   

   View Slide

7. Source
   code
   Scanner Parser
   @bsideup
   

   View Slide

8. Source
   code
   Scanner Parser
   Tokens
   @bsideup
   

   View Slide

9. Source
   code
   Scanner Parser
   Semantic
   analyzer
   Tokens
   @bsideup
   

   View Slide

10. Source
    code
    Scanner Parser
    Semantic
    analyzer
    Tokens CST

    (concrete syntax tree)
    @bsideup
    

    View Slide

11. Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Tokens CST

    (concrete syntax tree)
    @bsideup
    

    View Slide

12. Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Tokens CST

    (concrete syntax tree)
    @bsideup
    

    View Slide

13. Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

14. Middle End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

15. Middle End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    code
    generator
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

16. Middle End Back End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    code
    generator
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

17. Middle End Back End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Intermediate
    language
    code
    code
    generator
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

18. @bsideup
    

    View Slide

19. @bsideup
    

    View Slide

20. Middle End Back End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Intermediate
    language
    code
    code
    generator
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    ?
    

    View Slide

21. @bsideup
    

    View Slide

22. Transpiller
    @bsideup
    

    View Slide

23. “Transpilers, or source-to-source compilers,
    are tools that read source code written in one
    programming language, and produce the
    equivalent code in another language that has a
    similar level of abstraction.”
    @bsideup
    

    View Slide

24. What about Lombok?
    

    View Slide

25. Project Lombok
    @Value
    public class ValueExample {
    String name;
    @Wither(AccessLevel.PACKAGE)
    @NonFinal
    int age;
    double score;
    protected String[] tags;
    @ToString(includeFieldNames = true)
    @Value(staticConstructor = “of")
    public static class Exercise {
    String name;
    T value;
    }
    }
    @bsideup
    

    View Slide

26. Project Lombok
    @Value
    public class ValueExample {
    String name;
    @Wither(AccessLevel.PACKAGE)
    @NonFinal
    int age;
    double score;
    protected String[] tags;
    @ToString(includeFieldNames = true)
    @Value(staticConstructor = “of")
    public static class Exercise {
    String name;
    T value;
    }
    }
    public final class ValueExample {
    private final String name;
    private int age;
    private final double score;
    protected final String[] tags;
    @java.beans.ConstructorProperties({"name", "age", "score",
    public ValueExample(String name, int age, double score, Str
    this.name = name;
    this.age = age;
    this.score = score;
    this.tags = tags;
    }
    public String getName() {
    return this.name;
    }
    public int getAge() {
    return this.age;
    } @bsideup
    

    View Slide

27. }
    public double getScore() {
    return this.score;
    }
    public String[] getTags() {
    return this.tags;
    }
    @java.lang.Override
    public boolean equals(Object o) {
    if (o == this) return true;
    if (!(o instanceof ValueExample)) return false;
    final ValueExample other = (ValueExample)o;
    final Object this$name = this.getName();
    final Object other$name = other.getName();
    if (this$name == null ? other$name != null : !this$name
    if (this.getAge() != other.getAge()) return false;
    if (Double.compare(this.getScore(), other.getScore()) !
    if (!Arrays.deepEquals(this.getTags(), other.getTags())
    return true;
    }
    @java.lang.Override
    public int hashCode() {
    final int PRIME = 59;
    int result = 1;
    final Object $name = this.getName();
    result = result * PRIME + ($name == null ? 43 : $name.h
    result = result * PRIME + this.getAge();
    final long $score = Double.doubleToLongBits(this.getSco
    result = result * PRIME + (int)($score >>> 32 ^ $score)
    result = result * PRIME + Arrays.deepHashCode(this.getT
    return result;
    }
    Project Lombok
    @bsideup
    @Value
    public class ValueExample {
    String name;
    @Wither(AccessLevel.PACKAGE)
    @NonFinal
    int age;
    double score;
    protected String[] tags;
    @ToString(includeFieldNames = true)
    @Value(staticConstructor = “of")
    public static class Exercise {
    String name;
    T value;
    }
    }
    o_o
    

    View Slide

28. int result = 1;
    final Object $name = this.getName();
    result = result * PRIME + ($name == null ? 43 : $name.h
    result = result * PRIME + this.getAge();
    final long $score = Double.doubleToLongBits(this.getSco
    result = result * PRIME + (int)($score >>> 32 ^ $score)
    result = result * PRIME + Arrays.deepHashCode(this.getT
    return result;
    }
    @java.lang.Override
    public String toString() {
    return "ValueExample(name=" + getName() + ", age=" + ge
    Arrays.deepToString(getTags()) + ")";
    }
    ValueExample withAge(int age) {
    return this.age == age ? this : new ValueExample(name,
    }
    public static final class Exercise {
    private final String name;
    private final T value;
    private Exercise(String name, T value) {
    this.name = name;
    this.value = value;
    }
    public static Exercise of(String name, T value)
    return new Exercise(name, value);
    }
    public String getName() {
    return this.name;
    }
    public T getValue() {
    Project Lombok
    @bsideup
    @Value
    public class ValueExample {
    String name;
    @Wither(AccessLevel.PACKAGE)
    @NonFinal
    int age;
    double score;
    protected String[] tags;
    @ToString(includeFieldNames = true)
    @Value(staticConstructor = “of")
    public static class Exercise {
    String name;
    T value;
    }
    }
    o_O
    

    View Slide

29. }
    public T getValue() {
    return this.value;
    }
    @java.lang.Override
    public boolean equals(Object o) {
    if (o == this) return true;
    if (!(o instanceof ValueExample.Exercise)) return f
    final Exercise> other = (Exercise>)o;
    final Object this$name = this.getName();
    final Object other$name = other.getName();
    if (this$name == null ? other$name != null : !this$
    final Object this$value = this.getValue();
    final Object other$value = other.getValue();
    if (this$value == null ? other$value != null : !thi
    return true;
    }
    @java.lang.Override
    public int hashCode() {
    final int PRIME = 59;
    int result = 1;
    final Object $name = this.getName();
    result = result * PRIME + ($name == null ? 43 : $na
    final Object $value = this.getValue();
    result = result * PRIME + ($value == null ? 43 : $v
    return result;
    }
    @java.lang.Override
    public String toString() {
    return "ValueExample.Exercise(name=" + getName() +
    }
    }
    }
    Project Lombok
    @bsideup
    @Value
    public class ValueExample {
    String name;
    @Wither(AccessLevel.PACKAGE)
    @NonFinal
    int age;
    double score;
    protected String[] tags;
    @ToString(includeFieldNames = true)
    @Value(staticConstructor = “of")
    public static class Exercise {
    String name;
    T value;
    }
    }
    O_O
    

    View Slide

30. Is it a transpiler?
    

    View Slide

31. It could be, but…
    

    View Slide

32. Middle End Back End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Intermediate
    language
    code
    code
    generator
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

33. Middle End Back End
    Front End
    Source
    code
    Scanner Parser
    Semantic
    analyzer
    Optimizer
    Intermediate
    language
    code
    code
    generator
    Tokens CST

    (concrete syntax tree)
    AST

    (abstract syntax tree)
    @bsideup
    

    View Slide

34. The Hitchhiker's Guide
    to javac
    

    View Slide

35. @bsideup
    https://openjdk.java.net/groups/compiler/doc/hhgtjavac/index.html
    

    View Slide

36. @bsideup
    

    View Slide

37. @bsideup
    

    View Slide

38. @bsideup
    

    View Slide

39. @bsideup
    

    View Slide

40. Don’t confuse with…
    

    View Slide

41. @bsideup
    

    View Slide

42. Middle End
    Optimizer
    @bsideup
    

    View Slide

43. Why?
    

    View Slide

44. @bsideup
    

    View Slide

45. View Slide

46. View Slide

47. Lower example
    var result = switch (args.length) {
    case 1 -> {
    yield "one";
    }
    case 2, 3 -> "two or three";
    default -> "idk";
    };
    @bsideup
    

    View Slide

48. Lower example
    var result = switch (args.length) {
    case 1 -> {
    yield "one";
    }
    case 2, 3 -> "two or three";
    default -> "idk";
    };
    String innerPublic = null;
    switch (args.length) {
    case 1: {
    innerPublic = "one";
    break;
    }
    case 2:
    case 3: {
    innerPublic = "two or three";
    break;
    }
    default: {
    innerPublic = "idk";
    break;
    }
    }
    final String result = innerPublic;
    @bsideup
    

    View Slide

49. What can it “lower”?
    • Switch expressions
    • “var”
    • Try-with-resources
    • Pattern matching in “instanceof”
    • Even records!*
    @bsideup
    

    View Slide

50. @bsideup
    

    View Slide

51. How it decides
    when to lower?
    

    View Slide

52. It doesn’t!
    

    View Slide

53. But…
    

    View Slide

54. /**
    * Models a feature of the Java programming language. Each feature can be associated with a
    * minimum source level, a maximum source level and a diagnostic fragment describing the feature,
    * which is used to generate error messages of the kind {@code feature XYZ not supported in source N}.
    */
    public enum Feature {
    DIAMOND(JDK7, Fragments.FeatureDiamond, DiagKind.NORMAL),
    MODULES(JDK9, Fragments.FeatureModules, DiagKind.PLURAL),
    EFFECTIVELY_FINAL_VARIABLES_IN_TRY_WITH_RESOURCES(JDK9, Fragments.FeatureVarInTryWithResources, DiagKind.PLURAL),
    DEPRECATION_ON_IMPORT(MIN, JDK8),
    POLY(JDK8),
    LAMBDA(JDK8, Fragments.FeatureLambda, DiagKind.PLURAL),
    METHOD_REFERENCES(JDK8, Fragments.FeatureMethodReferences, DiagKind.PLURAL),
    DEFAULT_METHODS(JDK8, Fragments.FeatureDefaultMethods, DiagKind.PLURAL),
    STATIC_INTERFACE_METHODS(JDK8, Fragments.FeatureStaticIntfMethods, DiagKind.PLURAL),
    STATIC_INTERFACE_METHODS_INVOKE(JDK8, Fragments.FeatureStaticIntfMethodInvoke, DiagKind.PLURAL),
    STRICT_METHOD_CLASH_CHECK(JDK8),
    EFFECTIVELY_FINAL_IN_INNER_CLASSES(JDK8),
    TYPE_ANNOTATIONS(JDK8, Fragments.FeatureTypeAnnotations, DiagKind.PLURAL),
    ANNOTATIONS_AFTER_TYPE_PARAMS(JDK8, Fragments.FeatureAnnotationsAfterTypeParams, DiagKind.PLURAL),
    REPEATED_ANNOTATIONS(JDK8, Fragments.FeatureRepeatableAnnotations, DiagKind.PLURAL),
    INTERSECTION_TYPES_IN_CAST(JDK8, Fragments.FeatureIntersectionTypesInCast, DiagKind.PLURAL),
    GRAPH_INFERENCE(JDK8),
    FUNCTIONAL_INTERFACE_MOST_SPECIFIC(JDK8),
    POST_APPLICABILITY_VARARGS_ACCESS_CHECK(JDK8),
    MAP_CAPTURES_TO_BOUNDS(MIN, JDK7),
    PRIVATE_SAFE_VARARGS(JDK9),
    DIAMOND_WITH_ANONYMOUS_CLASS_CREATION(JDK9, Fragments.FeatureDiamondAndAnonClass, DiagKind.NORMAL),
    UNDERSCORE_IDENTIFIER(MIN, JDK8),
    PRIVATE_INTERFACE_METHODS(JDK9, Fragments.FeaturePrivateIntfMethods, DiagKind.PLURAL),
    LOCAL_VARIABLE_TYPE_INFERENCE(JDK10),
    VAR_SYNTAX_IMPLICIT_LAMBDAS(JDK11, Fragments.FeatureVarSyntaxInImplicitLambda, DiagKind.PLURAL),
    IMPORT_ON_DEMAND_OBSERVABLE_PACKAGES(JDK1_2, JDK8),
    SWITCH_MULTIPLE_CASE_LABELS(JDK14, Fragments.FeatureMultipleCaseLabels, DiagKind.PLURAL),
    SWITCH_RULE(JDK14, Fragments.FeatureSwitchRules, DiagKind.PLURAL),
    SWITCH_EXPRESSION(JDK14, Fragments.FeatureSwitchExpressions, DiagKind.PLURAL),
    TEXT_BLOCKS(JDK14, Fragments.FeatureTextBlocks, DiagKind.PLURAL),
    PATTERN_MATCHING_IN_INSTANCEOF(JDK14, Fragments.FeaturePatternMatchingInstanceof, DiagKind.NORMAL),
    REIFIABLE_TYPES_INSTANCEOF(JDK14, Fragments.FeatureReifiableTypesInstanceof, DiagKind.PLURAL),
    RECORDS(JDK14, Fragments.FeatureRecords, DiagKind.PLURAL),
    ;
    

    View Slide

55. PRIVATE_SAFE_VARARGS(JDK9),
    DIAMOND_WITH_ANONYMOUS_CLASS_CREATION(JDK9, Fragments.FeatureDiamondAndAnonClass, DiagKind.NORMAL),
    UNDERSCORE_IDENTIFIER(MIN, JDK8),
    PRIVATE_INTERFACE_METHODS(JDK9, Fragments.FeaturePrivateIntfMethods, DiagKind.PLURAL),
    LOCAL_VARIABLE_TYPE_INFERENCE(JDK10),
    VAR_SYNTAX_IMPLICIT_LAMBDAS(JDK11, Fragments.FeatureVarSyntaxInImplicitLambda, DiagKind.PLURAL),
    IMPORT_ON_DEMAND_OBSERVABLE_PACKAGES(JDK1_2, JDK8),
    SWITCH_MULTIPLE_CASE_LABELS(JDK14, Fragments.FeatureMultipleCaseLabels, DiagKind.PLURAL),
    SWITCH_RULE(JDK14, Fragments.FeatureSwitchRules, DiagKind.PLURAL),
    SWITCH_EXPRESSION(JDK14, Fragments.FeatureSwitchExpressions, DiagKind.PLURAL),
    TEXT_BLOCKS(JDK14, Fragments.FeatureTextBlocks, DiagKind.PLURAL),
    PATTERN_MATCHING_IN_INSTANCEOF(JDK14, Fragments.FeaturePatternMatchingInstanceof, DiagKind.NORMAL),
    REIFIABLE_TYPES_INSTANCEOF(JDK14, Fragments.FeatureReifiableTypesInstanceof, DiagKind.PLURAL),
    RECORDS(JDK14, Fragments.FeatureRecords, DiagKind.PLURAL),
    

    View Slide

56. PRIVATE_SAFE_VARARGS(JDK9),
    DIAMOND_WITH_ANONYMOUS_CLASS_CREATION(JDK9, Fragments.FeatureDiamondAndAnonClass, DiagKind.NORMAL),
    UNDERSCORE_IDENTIFIER(MIN, JDK8),
    PRIVATE_INTERFACE_METHODS(JDK9, Fragments.FeaturePrivateIntfMethods, DiagKind.PLURAL),
    LOCAL_VARIABLE_TYPE_INFERENCE(JDK10),
    VAR_SYNTAX_IMPLICIT_LAMBDAS(JDK11, Fragments.FeatureVarSyntaxInImplicitLambda, DiagKind.PLURAL),
    IMPORT_ON_DEMAND_OBSERVABLE_PACKAGES(JDK1_2, JDK8),
    SWITCH_MULTIPLE_CASE_LABELS(JDK14, Fragments.FeatureMultipleCaseLabels, DiagKind.PLURAL),
    SWITCH_RULE(JDK14, Fragments.FeatureSwitchRules, DiagKind.PLURAL),
    SWITCH_EXPRESSION(JDK14, Fragments.FeatureSwitchExpressions, DiagKind.PLURAL),
    TEXT_BLOCKS(JDK14, Fragments.FeatureTextBlocks, DiagKind.PLURAL),
    PATTERN_MATCHING_IN_INSTANCEOF(JDK14, Fragments.FeaturePatternMatchingInstanceof, DiagKind.NORMAL),
    REIFIABLE_TYPES_INSTANCEOF(JDK14, Fragments.FeatureReifiableTypesInstanceof, DiagKind.PLURAL),
    RECORDS(JDK14, Fragments.FeatureRecords, DiagKind.PLURAL),
    

    View Slide

57. PRIVATE_SAFE_VARARGS(JDK9),
    DIAMOND_WITH_ANONYMOUS_CLASS_CREATION(JDK9, Fragments.FeatureDiamondAndAnonClass, DiagKind.NORMAL),
    UNDERSCORE_IDENTIFIER(MIN, JDK8),
    PRIVATE_INTERFACE_METHODS(JDK9, Fragments.FeaturePrivateIntfMethods, DiagKind.PLURAL),
    LOCAL_VARIABLE_TYPE_INFERENCE(JDK10),
    VAR_SYNTAX_IMPLICIT_LAMBDAS(JDK11, Fragments.FeatureVarSyntaxInImplicitLambda, DiagKind.PLURAL),
    IMPORT_ON_DEMAND_OBSERVABLE_PACKAGES(JDK1_2, JDK8),
    SWITCH_MULTIPLE_CASE_LABELS(JDK14, Fragments.FeatureMultipleCaseLabels, DiagKind.PLURAL),
    SWITCH_RULE(JDK14, Fragments.FeatureSwitchRules, DiagKind.PLURAL),
    SWITCH_EXPRESSION(JDK14, Fragments.FeatureSwitchExpressions, DiagKind.PLURAL),
    TEXT_BLOCKS(JDK14, Fragments.FeatureTextBlocks, DiagKind.PLURAL),
    PATTERN_MATCHING_IN_INSTANCEOF(JDK14, Fragments.FeaturePatternMatchingInstanceof, DiagKind.NORMAL),
    REIFIABLE_TYPES_INSTANCEOF(JDK14, Fragments.FeatureReifiableTypesInstanceof, DiagKind.PLURAL),
    RECORDS(JDK14, Fragments.FeatureRecords, DiagKind.PLURAL),
    What if we…
    

    View Slide

58. PRIVATE_SAFE_VARARGS(JDK8),
    DIAMOND_WITH_ANONYMOUS_CLASS_CREATION(JDK8, Fragments.FeatureDiamondAndAnonClass, DiagKind.NORMAL),
    UNDERSCORE_IDENTIFIER(MIN, JDK8),
    PRIVATE_INTERFACE_METHODS(JDK9, Fragments.FeaturePrivateIntfMethods, DiagKind.PLURAL),
    LOCAL_VARIABLE_TYPE_INFERENCE(JDK8),
    VAR_SYNTAX_IMPLICIT_LAMBDAS(JDK8, Fragments.FeatureVarSyntaxInImplicitLambda, DiagKind.PLURAL),
    IMPORT_ON_DEMAND_OBSERVABLE_PACKAGES(JDK1_2, JDK8),
    SWITCH_MULTIPLE_CASE_LABELS(JDK8, Fragments.FeatureMultipleCaseLabels, DiagKind.PLURAL),
    SWITCH_RULE(JDK8, Fragments.FeatureSwitchRules, DiagKind.PLURAL),
    SWITCH_EXPRESSION(JDK8, Fragments.FeatureSwitchExpressions, DiagKind.PLURAL),
    TEXT_BLOCKS(JDK8, Fragments.FeatureTextBlocks, DiagKind.PLURAL),
    PATTERN_MATCHING_IN_INSTANCEOF(JDK8, Fragments.FeaturePatternMatchingInstanceof, DiagKind.NORMAL),
    REIFIABLE_TYPES_INSTANCEOF(JDK8, Fragments.FeatureReifiableTypesInstanceof, DiagKind.PLURAL),
    RECORDS(JDK8, Fragments.FeatureRecords, DiagKind.PLURAL),
    

    View Slide

59. How?
    

    View Slide

60. let’s instrument it!
    

    View Slide

61. void checkSourceLevel(Feature feature) {
    checkSourceLevel(token.pos, feature);
    }
    @bsideup
    

    View Slide

62. void checkSourceLevel(Feature feature) {
    if (feature.allowedInSource(Source.JDK8)) {
    feature = Source.Feature.LAMBDA;
    }
    checkSourceLevel(token.pos, feature);
    }
    @bsideup
    

    View Slide

63. Field field = Source.Feature.class.getDeclaredField("minLevel");
    field.setAccessible(true);
    for (Source.Feature feature : ENABLED_FEATURES) {
    field.set(feature, Source.JDK8);
    if (!feature.allowedInSource(Source.JDK8)) {
    throw new IllegalStateException(feature.name() + " minLevel instrumentation failed!");
    }
    }
    @bsideup
    

    View Slide

64. Field field = Source.Feature.class.getDeclaredField("minLevel");
    field.setAccessible(true);
    for (Source.Feature feature : ENABLED_FEATURES) {
    field.set(feature, Source.JDK8);
    if (!feature.allowedInSource(Source.JDK8)) {
    throw new IllegalStateException(feature.name() + " minLevel instrumentation failed!");
    }
    }
    @bsideup
    

    View Slide

65. Set ENABLED_FEATURES = Stream
    .of(
    "PRIVATE_SAFE_VARARGS",
    "SWITCH_EXPRESSION",
    "SWITCH_RULE",
    "SWITCH_MULTIPLE_CASE_LABELS",
    "LOCAL_VARIABLE_TYPE_INFERENCE",
    "VAR_SYNTAX_IMPLICIT_LAMBDAS",
    "DIAMOND_WITH_ANONYMOUS_CLASS_CREATION",
    "EFFECTIVELY_FINAL_VARIABLES_IN_TRY_WITH_RESOURCES",
    "TEXT_BLOCKS",
    "PATTERN_MATCHING_IN_INSTANCEOF",
    "REIFIABLE_TYPES_INSTANCEOF",
    "RECORDS"
    )
    .map(name -> {
    try {
    return Source.Feature.valueOf(name);
    } catch (IllegalArgumentException e) {
    return null;
    }
    })
    .filter(Objects::nonNull)
    .collect(Collectors.toSet());
    @bsideup
    

    View Slide

66. Demo
    

    View Slide

67. Takeaways
    • http://github.com/bsideup/jabel
    • Easy to integrate
    
    • Reuses compiler’s code (except for records)
    
    • Pretty safe to use

    @bsideup
    

    View Slide

68. @bsideup
    bsideup
    

    View Slide