Everything is broken, and I don't know why. Python edition.

PyOhio
July 30th 2017

Transcript

1. Everything is broken,
   and I don’t know why.
   Python edition
   PyOhio
   July 30th 2017
   Matt Robenolt
   

   View Slide

2. hello @mattrobenolt
   

   View Slide

3. Core contributor
   

   View Slide

4. If an exception happens in
   production, and nobody sees
   the logs, did it really happen?
   — Taylor Swift
   “
   ”
   

   View Slide

5. Damn Stebe, it works for me.
   gtfo my inbox.
   “
   ”
   — Josh
   

   View Slide

6. View Slide

7. But first,
   some broken software.
   

   View Slide

8. Errors in: Computers
   EAX
   CPU registers
   EBX
   ECX
   

   View Slide

9. Errors in: Computers
   EAX
   CPU registers
   EBX
   SYS_open
   ECX
   

   View Slide

10. Errors in: Computers
    EAX
    CPU registers
    EBX
    SYS_open
    ECX
    "foo.txt"
    

    View Slide

11. Errors in: Computers
    EAX
    CPU registers
    EBX
    SYS_open
    ECX
    "foo.txt"
    O_RDONLY
    

    View Slide

12. Errors in: Computers
    EAX
    CPU registers
    EBX
    ECX
    "foo.txt"
    interrupt
    SYS_open
    O_RDONLY
    

    View Slide

13. Errors in: Computers
    EAX
    CPU registers
    EBX
    ECX
    "foo.txt"
    interrupt
    EAX -1
    SYS_open
    O_RDONLY
    

    View Slide

14. #include
    int main () {
    FILE *fp;
    char buf[100];
    fp = fopen("foo.txt", "r");
    fgets(buf, 100, fp);
    fclose(fp);
    }
    Errors in: C
    

    View Slide

15. #include
    int main () {
    FILE *fp;
    char buf[100];
    fp = fopen("foo.txt", "r");
    fgets(buf, 100, fp);
    fclose(fp);
    }
    Errors in: C
    [1] 53482 segmentation fault ./a.out
    

    View Slide

16. #include
    int main () {
    FILE *fp;
    char buf[100];
    fp = fopen("foo.txt", “r");
    if (fp != NULL) {
    fgets(buf, 100, fp);
    fclose(fp);
    }
    }
    Errors in: C
    

    View Slide

17. package main
    import "os"
    func main() {
    buf := make([]byte, 100)
    fp, _ := os.Open("foo.txt")
    fp.Read(buf)
    fp.Close()
    }
    Errors in: Go
    

    View Slide

18. package main
    import "os"
    func main() {
    buf := make([]byte, 100)
    fp, _ := os.Open("foo.txt")
    fp.Read(buf)
    fp.Close()
    }
    Errors in: Go
    jk nothing happens
    

    View Slide

19. package main
    import "os"
    func main() {
    buf := make([]byte, 100)
    fp, err := os.Open(“foo.txt")
    if err != nil {
    return
    }
    fp.Read(buf)
    fp.Close()
    }
    Errors in: Go
    

    View Slide

20. use std::fs::File;
    use std::io::Read;
    fn main() {
    let mut fp = File::open("foo.txt").unwrap();
    let mut buf = [0; 100];
    fp.read_exact(&mut buf);
    drop(fp);
    }
    Errors in: Rust
    

    View Slide

21. use std::fs::File;
    use std::io::Read;
    fn main() {
    let mut fp = File::open("foo.txt").unwrap();
    let mut buf = [0; 100];
    fp.read_exact(&mut buf);
    drop(fp);
    }
    Errors in: Rust
    thread 'main' panicked at 'called `Result::unwrap()`
    on an `Err` value: Error { repr: Os { code: 2,
    message: "No such file or directory" } }', ../src/
    libcore/result.rs:788
    note: Run with `RUST_BACKTRACE=1` for a backtrace.
    

    View Slide

22. use std::fs::File;
    use std::io::Read;
    fn main() {
    match File::open("foo.txt") {
    Ok(mut fp) => {
    let mut buf = [0; 100];
    fp.read_exact(&mut buf);
    drop(fp);
    },
    Err(_) => {
    //
    },
    }
    }
    Errors in: Rust
    

    View Slide

23. fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    Errors in: Python
    

    View Slide

24. fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    Errors in: Python
    Traceback (most recent call last):
    File "main.py", line 1, in
    fp = open('foo.txt', 'r')
    IOError: [Errno 2] No such file or directory:
    'foo.txt'
    

    View Slide

25. try:
    fp = open('foo.txt', 'r')
    except IOError:
    pass
    else:
    buf = fp.read(100)
    fp.close()
    Errors in: Python
    

    View Slide

26. try:
    fp = open('foo.txt', 'r')
    except IOError:
    pass
    else:
    buf = fp.read(100)
    fp.close()
    Errors in: Python
    Exceptions
    

    View Slide

27. Exceptions
    Errors
    vs
    

    View Slide

28. Errors
    

    View Slide

29. Errors
    Non-fatal, multiple may exist within
    a call stack at the same time,
    generally returned as a value.
    

    View Slide

30. Exceptions
    Fatal, must be handled, only one
    exists within a call stack, raised to
    abort current execution.
    

    View Slide

31. Exceptions
    Fatal, must be handled, only one
    exists within a call stack, raised to
    abort current execution.
    

    View Slide

32. Traceback (most recent call last):
    File "main.py", line 1, in
    fp = open('foo.txt', 'r')
    IOError: [Errno 2] No such file or directory:
    'foo.txt'
    

    View Slide

33. Traceback (most recent call last):
    File "main.py", line 1, in
    fp = open('foo.txt', 'r')
    IOError: [Errno 2] No such file or directory:
    'foo.txt'
    So what do we do with this thing?
    

    View Slide

34. Traceback (most recent call last):
    File "main.py", line 1, in
    fp = open('foo.txt', 'r')
    IOError: [Errno 2] No such file or directory:
    'foo.txt'
    So what do we do with this thing?
    …and where does it come from?
    

    View Slide

35. fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    

    View Slide

36. import sys
    def excepthook(type, value, tb):
    print(type, value, tb)
    sys.excepthook = excepthook
    fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    

    View Slide

37. import sys
    def excepthook(type, value, tb):
    print(type, value, tb)
    sys.excepthook = excepthook
    fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    (, IOError(2, 'No such
    file or directory'), 0x106810ea8>)
    

    View Slide

38. import sys
    def excepthook(type, value, tb):
    import traceback
    traceback.print_exception(type, value, tb)
    sys.excepthook = excepthook
    fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    

    View Slide

39. import sys
    def excepthook(type, value, tb):
    import traceback
    traceback.print_exception(type, value, tb)
    sys.excepthook = excepthook
    fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    Traceback (most recent call last):
    File "main.py", line 9, in
    fp = open('foo.txt', 'r')
    IOError: [Errno 2] No such file or directory:
    'foo.txt'
    

    View Slide

40. fp = open('foo.txt', 'r')
    buf = fp.read(100)
    fp.close()
    

    View Slide

41. try:
    fp = open('foo.txt', 'r')
    except IOError:
    import sys
    print(sys.exc_info())
    else:
    buf = fp.read(100)
    fp.close()
    

    View Slide

42. try:
    fp = open('foo.txt', 'r')
    except IOError:
    import sys
    print(sys.exc_info())
    else:
    buf = fp.read(100)
    fp.close()
    (, IOError(2, 'No such
    file or directory'), 0x106810ea8>)
    

    View Slide

43. try:
    fp = open('foo.txt', 'r')
    except IOError:
    import sys, traceback
    traceback.print_exception(*sys.exc_info())
    else:
    buf = fp.read(100)
    fp.close()
    

    View Slide

44. try:
    fp = open('foo.txt', ‘r')
    except IOError:
    import traceback
    traceback.print_exc()
    else:
    buf = fp.read(100)
    fp.close()
    Traceback (most recent call last):
    File "main.py", line 2, in
    fp = open('foo.txt', 'r')
    IOError: [Errno 2] No such file or directory:
    'foo.txt'
    

    View Slide

45. import sys
    print(sys.exc_info())
    

    View Slide

46. import sys
    print(sys.exc_info())
    (None, None, None)
    

    View Slide

47. import sys
    try:
    raise Exception()
    except Exception:
    pass
    print(sys.exc_info())
    

    View Slide

48. import sys
    try:
    raise Exception()
    except Exception:
    pass
    print(sys.exc_info())
    (None, None, None)
    

    View Slide

49. try:
    raise Exception()
    except Exception:
    import sys
    print(sys.exc_info())
    

    View Slide

50. try:
    raise Exception()
    except Exception:
    import sys
    print(sys.exc_info())
    (, Exception(),
    )
    

    View Slide

51. try:
    raise Exception()
    except Exception:
    import traceback, sys
    traceback.print_exc(file=sys.stderr)
    

    View Slide

52. $ python main.py 2> errors.log
    

    View Slide

53. View Slide

54. $ cat errors.log
    

    View Slide

55. $ cat errors.log
    raise Exception()
    Exception
    Traceback (most recent call last):
    File "main.py", line 3, in
    raise Exception()
    Exception
    Traceback (most recent call last):
    File "main.py", line 3, in
    raise Exception()
    Exception
    Traceback (most recent call last):
    File "main.py", line 3, in
    raise Exception()
    Exception
    Traceback (most recent call last):
    File "main.py", line 3, in
    

    View Slide

56. $ cat errors.log
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing1'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing2'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing3'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing4'
    

    View Slide

57. $ cat errors.log
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing1'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing2'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing3'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing4'
    

    View Slide

58. Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing1'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing2'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing3'
    Traceback (most recent call last):
    File "main.py", line 4, in
    my_thing = things[key]
    KeyError: 'thing4'
    

    View Slide

59. Damn Stebe, that
    variable should be
    right. You must
    have messed it up.
    “
    ”
    — Darrell
    

    View Slide

60. things = {}
    key = sys.argv[1]
    try:
    my_thing = things[key]
    except KeyError:
    import traceback
    traceback.print_exc()
    print(globals(), locals())
    

    View Slide

61. things = {}
    key = sys.argv[1]
    try:
    my_thing = things[key]
    except KeyError:
    import traceback
    traceback.print_exc()
    print(globals(), locals())
    

    View Slide

62. things = {}
    key = sys.argv[1]
    try:
    my_thing = things[key]
    except KeyError:
    import traceback
    traceback.print_exc()
    print(globals(), locals())
    Traceback (most recent call last):
    File "main.py", line 5, in
    things[key]
    KeyError: 'thing'
    ({'things': {},'key': 'thing'}, {'things':
    {}, 'key': 'thing'})
    

    View Slide

63. things = {}
    key = sys.argv[1]
    try:
    my_thing = things[key]
    except KeyError:
    import traceback
    traceback.print_exc()
    print(globals(), locals())
    Traceback (most recent call last):
    File "main.py", line 5, in
    things[key]
    KeyError: 'thing'
    ({'things': {},'key': 'thing'}, {'things':
    {}, 'key': 'thing'})
    

    View Slide

64. things = {}
    key = sys.argv[1]
    try:
    my_thing = things[key]
    except KeyError:
    import traceback, json, sys
    json.dump({
    'exc': traceback.format_exc(),
    'globals': globals(),
    'locals': locals(),
    }, sys.stderr)
    Don’t actually do this, use:
    https://github.com/hynek/structlog
    

    View Slide

65. things = [0, 1, 2]
    def get_random_thing(n):
    return get_thing(randint(0, n))
    def get_thing(index):
    try:
    return things[index]
    except IndexError:
    print(locals())
    get_random_thing(10)
    

    View Slide

66. things = [0, 1, 2]
    def get_random_thing(n):
    return get_thing(randint(0, n))
    def get_thing(index):
    try:
    return things[index]
    except IndexError:
    print(locals())
    get_random_thing(10)
    {'index': 8}
    

    View Slide

67. def debug():
    import sys
    tb = sys.exc_info()[2]
    idx = 0
    while tb:
    idx += 1
    frame = tb.tb_frame
    code = frame.f_code
    print({
    'line': tb.tb_lineno,
    'locals': frame.f_locals,
    'function': code.co_name,
    })
    tb = tb.tb_next
    try:
    get_random_thing(10)
    except Exception:
    debug()
    

    View Slide

68. def debug():
    import sys
    tb = sys.exc_info()[2]
    idx = 0
    while tb:
    idx += 1
    frame = tb.tb_frame
    code = frame.f_code
    print({
    'line': tb.tb_lineno,
    'locals': frame.f_locals,
    'function': code.co_name,
    })
    tb = tb.tb_next
    try:
    get_random_thing(10)
    except Exception:
    debug()
    {'function': ‘',
    'line': 31,
    'locals': {'things': [0, 1, 2]}}
    {'function': 'get_random_thing',
    'line': 7,
    'locals': {'n': 4}}
    {'function': 'get_thing',
    'line': 11,
    'locals': {'index': 4}}
    

    View Slide

69. def debug():
    report = {
    'context': {
    'argv': sys.argv,
    'environ': os.environ,
    'uname': os.uname(),
    'time': time.time(),
    },
    'frames': [],
    }
    

    View Slide

70. _, e, tb = sys.exc_info()
    idx = 0
    while tb:
    idx += 1
    frame = tb.tb_frame
    code = frame.f_code
    report['frames'].append({
    'line': tb.tb_lineno,
    'locals': frame.f_locals,
    'function': code.co_name,
    'filename': code.co_filename,
    'context': linecache.getline(
    code.co_filename,
    tb.tb_lineno,
    ),
    })
    tb = tb.tb_next
    

    View Slide

71. sendmail('m@robenolt.com',
    e.message, serialize(report))
    

    View Slide

72. Hot damn Stebe,
    thanks for the
    information. Lemme fix
    that bug for you right
    away.
    “
    ”
    

    View Slide

73. Sentry.
    We do all this shit for you.
    

    View Slide

74. Questions?
    I may or may not have answers.
    @mattrobenolt
    github.com/mattrobenolt
    sentry.io
    github.com/getsentry/sentry
    

    View Slide