Abstracting Failure Inducing Inputs

ISSTA 2020

Transcript

1. Abstracting Failure Inducing Inputs Rahul Gopinath Alexander Kampmann Nikolas Havrikov Ezekiel Soremekun Andreas Zeller CISPA Helmholtz Center for Information Security
2. Abstracting Failure Inducing Inputs Rahul Gopinath Alexander Kampmann Nikolas Havrikov Ezekiel Soremekun Andreas Zeller CISPA Helmholtz Center for Information Security
3. 3 (1 + 2 + 334) ✓ Program
4. 4 (1 + 2 + 334) Program (( 442 / 3 )) - 1 ✘
5. 5 (1 + 2 + 334) Program (( 442 / 3 )) - 1 ✓ 2 -( 19 - 34 ) + 9 - 7
6. 6 (1 + 2 + 334) Program (( 442 / 3 )) - 1 2 -( 19 - 34 ) + 9 - 7 (1) + ((3 + 1 / 334)) + 2 ✘
7. 7 (1 + 2 + 334) (( 442 / 3 )) - 1 2 -( 19 - 34 ) + 9 - 7 (1) + ((3 + 1 / 334)) + 2 if '((' in input and '))' in input: raise Exception() Program
8. 8 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ✘
9. Why did my program fail?
10. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program
11. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ?
12. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ?
13. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ?
14. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ?
15. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ( ( ) ) 4
16. Delta Debugging 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Program ✘ (( )) 4 Reproduced the failure
17. Context Free Grammar Structured Inputs See also: "Learning Input Tokens for Effective Fuzzing" ISSTA '20 https://www.slideshare.net/BjrnMathis/lfuzzer-learning-input-tokens-for-effective-fuzzing-237085021
18. 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Interpreter ✘ Reproduced the failure Structured Inputs SYNTAX CHECK
19. 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Interpreter Structured Inputs SYNTAX ERROR #
20. 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) + 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a-+(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+- +-7 * (-0 * (+(((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / +++6.37) + (1) / 482) / +++-+0)))) * - +5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 + * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * 8.2 - 27 - -9 / +((+9 * --2 + --+-+-((-1 * +(8 - 5 - 6)) * (-(a- +(((+(4))))) - ++4) / +(-+---((5.6 - --(3 * -1.8 * +(6 * +-(((-(-6) * ---+6)) / +--(+-+-7 * (-0 * (+ (((((2)) + 8 - 3 - ++9.0 + ---(--+7 / (1 / ++ +6.37) + (1) / 482) / +++-+0)))) * -+5 + 7.513)))) - (+1 / ++((-84)))))))) * ++5 / +-(--2 - -++-9.0)))) / 5 * --++090 ++5 / +-(--2 - -+ +-9.0)))) / 5 * --++090 Interpreter Structured Inputs SYNTAX ERROR #
21. SYNTAX ERROR
22. Solution: Work on the Parse Tree
23. 23 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] 1 + (( 2 * 3 / 4 ))
24. 24 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] 1 + (( 2 * 3 / 4 ))
25. 25 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] 1 + (( 2 * 3 / 4 )) ✘
26. 26 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] (( 2 * 3 / 4 )) ✘
27. 27 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] (( 3 / 4 )) ✘
28. 28 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] (( 4 )) ✘
29. 29 ( 1 + (( 2 * 3 / 4 )) ( 4 ))
30. 30 ( 1 + (( 2 * 3 / 4 )) (4)) What caused this failure? Does the failure occur in other inputs?
31. 31 ( 1 + (( 2 * 3 / 4 )) (4)) 4444 ? ()() ? ((-4)) ? ((29)) ? ((v)) ? +++1 ?
32. 32 var A = class extends (class {}){}; Issue 2937 from Closure
33. 33 { while ((l_0)){ if ((l_0)) { break;;var l_0; continue }0 } } Issue 2842 from Closure
34. 34 var {baz:{} = baz => {}} = baz => {}; Issue 385 from Rhino
35. 35 const [y,y] = []; Issue 386 from Rhino
36. 36 Abstraction with DDSet
37. 37 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9]
38. 38 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] ✓ Did not reproduce the failure 1 * (2 - 3)
39. 39 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9]
40. 40 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] c
41. 41 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] c ✓ Did not reproduce the failure 1 + 3 + 4
42. 42 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] c c
43. 43 3 * 4 <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] c c ✓ Did not reproduce the failure
44. 44 ( ( 4 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] c c c c c c c
45. 45 ( ( 1 - 2 ) ) <start> := <expr> <expr> := <term> ' + ' <expr> | <term> ' - ' <expr> | <term> <term> := <factor> ' * ' <term> | <factor> ' / ' <term> | <factor> <factor> := '+' <factor> | '-' <factor> | '(' <expr> ')' | <integer> '.' <integer> | <integer> <integer>:= <digit> <integer> | <digit> <digit> := [0-9] c c c c c c c ✘ reproduced the failure ( ( 1 - 2 ) )
46. 46 ( ( 1 - 2 ) ) c c c c c c c ✘ ( ( 1 - 2 ) )
47. 47 ( ( 1 - 2 ) ) c c c c c c c ✘ ( ( 1 - 2 ) ) ✘ ( ( 2 * 3 + 4 ) )
48. 48 ( ( 1 - 2 ) ) c c c c c c c ✘ ( ( 1 - 2 ) ) ✘ ( ( 2 * 3 + 4 ) ) ✘ ( ( - 2 / 1 ) )
49. 49 ( ( 1 - 2 ) ) c c c c c c c ✘ ( ( 1 - 2 ) ) ✘ ( ( 2 * 3 + 4 ) ) ✘ ( ( - 2 / 1 ) ) ✘ ( ( 98 - 0 ) )
50. <expr> ) ( ( ) ( ( ) 4 ) 50 ( ( 4 ) ) c c c c c c c A
51. <expr> ) ( ( ) ( ( ) 4 ) 51 ( ( 4 ) ) c c c c c c c A
52. 52 ( ( 4 ) ) c c c c c c c A ( ( ) ) <expr> ( ( ) ) 4 Minimized Input Abstract Failure Inducing Input ((1 + 2)) ((23 * 3 - 34)) ((344- 4 + (223))) (((1) - 3 * 773 + (-22 + 1))) ((1798 - 889 / (333-1) * 2 / 3 + 1)) ((34 + 4 --334 + (334 - (22) + 919 * 0 + 1)) ((98435747+ 88 + (((0))) + (1) - 1 * 7 / 4 * 889 - 2)) ((8 + ((8)) + --1 + 11223 / 344 - 39 + (1) - 456 + 134 / 45 )) ((437 + 8 - 1 * ((9 + 1) - 1 + 99111948 + 3 --1 + (112) - 2 + 445) + 0)) ((74 + 334 + ((178 - 88 / (3393-1) * 1002 / 3 + 1+ 3439)) * 223 - 1233 + 334672 (( 2 * 9 - (1798 - 889 / (333-1) * 2 / 3 + 100012 + 3434392 + 234 ----6 * 1798 - 889 / (333 ((778 - (((1) - 3 * 773 + (-22 + 1) * (4545) - 23 - (2) * 773 + (-22 + 1) / 3434 + ---1 + 1 / 34343 + 112 ((349 + (((1) - 3 * 3 + (-22 + 1) ((+ (-22 + 1) * (4545) - 23 - (2) * 773 + (-22 + 1) / 3434 + ---1 + 1 / 34343 + 1123 ((8 + ((8)) + --1 + / 1 - 39 + (1) - 456 + 134 / 45 ))(((1) - 2334 + (((1) - 3 * 773 + (-22 + 1) * (2) - 23 - (2) * 773 + (-22 + 1) / 3 ((74 + 3 + ((178 - 88 / (3393-1) * 1002 / 3 + 1+ 3439)) * - 1233 + 334672)) ((8 + ((8)) + --1 + / 344 - 39 + (1) - 456 + 134 / 45 ))(((1) - 3 * 77 ((1+ 33+ 24343433 +23343 - ((74 + 334 + ((178 - 88 / (3393-1) * 1002 / 3 + 1+ 3439)) * - 1233 + 334672)) ((8 + ((8)) + --1 + / 344 - 39 + (1) - 456 + 134 / ✘ ✘
53. <varModifier> <Identifier> = class extends (class {}){} 53 var A = class extends (class {}){}; Issue 2937 from Closure
54. 54 var A = class extends (class {}){}; Issue 2937 from Closure <varModifier> <Identifier> = class extends (class {}){}
55. var {baz:{} = baz => {}} = <variableDeclaration> 55 var {baz:{} = baz => {}} = baz => {}; Issue 385 from Rhino
56. 56 var {baz:{} = baz => {}} = baz => {}; Issue 385 from Rhino var {baz:{} = baz => {}} = <variableDeclaration>
57. const [y,y] = []; 57 const [y,y] = []; Issue 386 from Rhino
58. 58 const [y,y] = []; Issue 386 from Rhino const [y,y] = [];
59. v = 0; v = v Co-varying Fragments
60. v = 0; v = v Co-varying Fragments
61. v = 0; v = v
62. v = 0; v = v x = 0; v = v ✓
63. v = 0; v = v v = 0; r = v ✓
64. v = 0; v = v v = 0; v = 0 ✓
65. v = 0; v = v • Identify matching nonterminals of concrete nodes • Modify them together
66. v = 0; v = v z = 0; z = z ✘ • Identify matching nonterminals of concrete nodes • Modify them together
67. v = 0; v = v z = 0; z = z ✘ • Identify matching nonterminals of concrete nodes • Modify them together p = 0; p = p ✘
68. v = 0; v = v z = 0; z = z ✘ • Identify matching nonterminals of concrete nodes • Modify them together p = 0; p = p ✘ c = 0; c = c ✘
69. v = 0; v = v <alpha1> = <expr>; <alpha1> = <alpha1> V1 V1 V1
70. const [y,y] = []; Issue 386 from Rhino
71. const [y,y] = []; Issue 386 from Rhino const [<$Id1>,<$Id1>] = []
72. 72 var {baz:{} = baz => {}} = baz => {}; Issue 385 from Rhino
73. 73 var {baz:{} = baz => {}} = baz => {}; Issue 385 from Rhino var {<$Id1>:{} = <$Id1> => {}} <variableDeclaration>;
74. 74 {while ((l_0)){ if ((l_0)) {break;;var l_0; continue }0}} Issue 2842 from Closure
75. 75 {while ((l_0)){ if ((l_0)) {break;;var l_0; continue }0}} Issue 2842 from Closure {while ((<$Id1>)){ if ((<$Id1>)) {break;;var <$Id1>; continue }0}}
76. Program %Valid %Fail lua-5.3.5 4 100.0 100.0 clj-2092 100.0 100.0 clj-2345 100.0 100.0 clj-2450 62.0 100.0 clj-2473 40.0 100.0 clj-2518 100.0 100.0 clj-2521 100.0 100.0 closure.1978 76.0 100.0 closure.2808 100.0 100.0 closure.2842 100.0 99.0 closure.2937 36.0 100.0 closure.3178 57.0 100.0 closure.3379 84.0 100.0 rhino 385 49.0 100.0 rhino 386 100.0 100.0 find 07b941b1 100.0 100.0 find 93623752 100.0 100.0 find c8491c11 100.0 100.0 find dbcb10e9 100.0 100.0 grep 3c3bdace 100.0 100.0 grep 54d55bba 100.0 100.0 grep 9c45c193 100.0 100.0 Mean 86.54 100.0 • Lua • lua-5.3.5 (1 bug) • Javascript • rhino-1.7.7.2 (2 bugs) • closure 20151216 (1 bug) • closure 20171203 (3 bugs) • closure 20200101 (2 bugs) • Clojure • clojure-1.11.0 (6 bug) • Unix Utilities (dbgbench) • find (4 bugs) • grep (3 bugs) Experimental Results
77. Where do the grammars come from? https://rahul.gopinath.org/post/2020/07/15/ddset/ https://github.com/vrthra/ddset Stay tuned for our FSE 2020 Paper Mining Input Grammars from Dynamic Control Flow
78. Future... Algebra of Behavior Inducing Patterns Grammar refinement: generate(Grefined): 123 + ((34 + 5)) - 244 After fixing a failure, I want to produce numerous new inputs that • Induces the same failure (R1) • Covers what I just checked in (R2) • Does not go through validation (R3) • ... Grefined = R1 & R2 & !R3 ...
79. https://rahul.gopinath.org/post/2020/07/15/ddset/ https://github.com/vrthra/ddset
80. 80 https://rahul.gopinath.org/post/2020/07/15/ddset/ https://github.com/vrthra/ddset