Upgrade to Pro — share decks privately, control downloads, hide ads and more …

My Python is Rusting

My Python is Rusting

Presentation at PyCon Cz 2017 about Rust and Python

Armin Ronacher

June 08, 2017
Tweet

More Decks by Armin Ronacher

Other Decks in Programming

Transcript

  1. MY PYTHON IS RUSTING
    Armin @mitsuhiko Ronacher
    — A PYTHON AND RUST LOVE STORY —

    View full-size slide

  2. )J *N"SNJO
    BOE*EP0QFO4PVSDF
    MPUTPG1ZUIPOBOE4BB4
    'MBTL
    4FOUSZ
    j

    View full-size slide

  3. … and here
    is where you
    can find me
    twitter.com/@mitsuhiko
    github.com/mitsuhiko
    lucumr.pocoo.org/

    View full-size slide

  4. “so I heard you are doing Rust now …”

    View full-size slide

  5. that's true but …

    View full-size slide

  6. we love python

    View full-size slide

  7. Strong Ecosystem

    View full-size slide

  8. Fast Iteration

    View full-size slide

  9. Stable Environment

    View full-size slide

  10. Powerful

    Metaprogramming

    View full-size slide

  11. Fast
    Interpreter
    Introspection

    View full-size slide

  12. Functionality

    View full-size slide

  13. what we use it for

    View full-size slide

  14. Mach-O / Dwarf Parsing

    View full-size slide

  15. Javascript Source Maps

    View full-size slide

  16. Proguard Mappings

    View full-size slide

  17. Command Line Tools

    View full-size slide

  18. one to the other

    View full-size slide

  19. virtualenv & pip & distutils & setuptools
    rustup & cargo

    View full-size slide

  20. rustup
    the rust toolchain manager

    View full-size slide

  21. cargo
    the rust package manager

    View full-size slide

  22. pydoc & sphinx
    rustdoc

    View full-size slide

  23. rustdoc
    the rust documentation builder

    View full-size slide

  24. a rust primer

    View full-size slide

  25. fn main() {
    println!("Hello World!");
    }

    View full-size slide

  26. use std::io::{stdin, BufRead, BufReader};
    use std::collections::HashMap;
    fn main() {
    let mut counts = HashMap::new();
    for line_rv in BufReader::new(stdin()).lines() {
    let line = line_rv.unwrap();
    *counts.entry(line).or_insert(0) += 1;
    }
    let mut items: Vec<_> = counts.into_iter().collect();
    items.sort_by_key(|&(_, count)| -count);
    for (item, count) in items.into_iter().take(10) {
    println!("{}: {}", item, count);
    }
    }

    View full-size slide

  27. use std::io::{stdin, BufRead, BufReader};
    use std::collections::HashMap;
    fn main() {
    let mut counts = HashMap::new();
    for line_rv in BufReader::new(stdin()).lines() {
    let line = line_rv.unwrap();
    *counts.entry(line).or_insert(0) += 1;
    }
    let mut items: Vec<_> = counts.into_iter().collect();
    items.sort_by_key(|&(_, count)| -count);
    for (item, count) in counts {
    println!("{}: {}", item, count);
    }
    }

    View full-size slide

  28. error[E0382]: use of moved value: `counts`
    --> test.rs:16:26
    |
    13 | let mut items: Vec<_> = counts.into_iter().collect();
    | ------ value moved here
    ...
    16 | for (item, count) in counts {
    | ^^^^^^ value used here after move
    |
    = note: move occurs because `counts` has type
    `std::collections::HashMap`,
    which does not implement the `Copy` trait

    View full-size slide

  29. use std::fmt;
    struct User {
    id: i64,
    name: String,
    }
    impl fmt::Display for User {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    write!(f, "", self.id, self.name)
    }
    }
    fn main() {
    println!("{}", User { id: 42, name: "Peter".to_string() });
    }

    View full-size slide

  30. the zen of python

    View full-size slide

  31. Beautiful is better than ugly.
    #[derive(Serialize, Deserialize, Debug)]
    pub struct Deploy {
    #[serde(rename="environment")]
    pub env: String,
    pub name: Option,
    pub url: Option,
    }
    impl Deploy {
    pub fn list(&self, api: &Api, id: u64) -> ApiResult> {
    api.get(&format!("/deploys/{}/", id))?.convert()
    }
    }

    View full-size slide

  32. Explicit is better than implicit.
    fn parse_rev_range(rng: &str) -> (Option<&str>, &str) {
    if rng == "" {
    return (None, "HEAD".into());
    }
    let mut iter = rng.rsplitn(2, "..");
    let rev = iter.next().unwrap_or("HEAD");
    (iter.next(), rev)
    }

    View full-size slide

  33. Simple is better than complex.
    use std::{fs, env, io};
    let here = env::current_dir()?;
    for dent_rv in fs::read_dir(here)? {
    let dent = dent_rv?;
    let md = dent.metadata()?;
    println!("{: <60}{: <12}{}",
    dent.path().display(),
    md.len(),
    if md.is_file() { "file" } else { "dir" });
    }

    View full-size slide

  34. Complex is better than complicated.
    use redis::{Client, PipelineCommands, pipe};
    let client = Client::open("redis://127.0.0.1/")?;
    let con = client.get_connection()?;
    let (k1, k2) : (i32, i32) = pipe()
    .atomic()
    .set("key_1", 42).ignore()
    .set("key_2", 43).ignore()
    .get("key_1")
    .get("key_2").query(&con)?;

    View full-size slide

  35. Errors should never pass silently.
    use std::fs;
    fn main() {
    fs::File::open("/tmp/test.txt");
    }
    $ rustc test.rs
    warning: unused result which must be used,
    #[warn(unused_must_use)] on by default
    --> test.rs:4:5
    |
    4 | fs::File::open("/tmp/test.txt");
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

    View full-size slide

  36. side by side

    View full-size slide

  37. protocols vs traits
    __del__
    __add__
    __str__
    __repr__
    __getitem__
    Drop::drop
    Add::add
    Display::fmt
    Debug::fmt
    Index::index

    View full-size slide

  38. error causing
    throw …
    panic!(…)
    return Err(…);

    View full-size slide

  39. error conversion
    try:
    x = foo()
    except SomeError as e:
    raise NewError(e)
    let x = foo()?

    View full-size slide

  40. so you want to try it

    View full-size slide

  41. [ Intermission ]

    View full-size slide

  42. monolithic SOA
    /

    View full-size slide

  43. this is absurd

    View full-size slide

  44. modular code
    same process
    +

    View full-size slide

  45. cffi you Python and Rust
    you declared your compatibility and now I

    View full-size slide

  46. Rust Library
    > Rust CABI + C header
    > CFFI
    > Python
    cargo | cffi | wheel | setuptools

    View full-size slide

  47. GO AWAY LIBPYTHON
    repeat after me:

    View full-size slide

  48. do start threads
    and thread pools

    View full-size slide

  49. don't pass complex

    data structures around

    View full-size slide

  50. do wrap some rust
    objects in python

    View full-size slide

  51. don't move complex logic

    from Python to Rust

    View full-size slide

  52. do use docker for builds

    View full-size slide

  53. Pillow-4.0.0-cp36-cp36m-manylinux1_x86_64.whl
    Python 2 builds: Python 3 builds:
    Versions: 2.7
    ABI: cpm + cpmu
    Platforms: OS X + 2 Linux
    Total: 1 ×2 × 3 = 6
    Versions: 3.3 + 3.4 + 3.5 + 3.6 + 3.7
    ABI: cpm
    Platforms: OS X + 2 Linux
    Total: 5 ×1 × 3 = 15

    View full-size slide

  54. 21 BUILDS!!!

    View full-size slide

  55. path to success:
    • do not link to libpython
    • use cffi
    • 2.x/3.x compatible sources
    • fuck around with setuptools

    View full-size slide

  56. symsynd-1.3.0-py2.py3-none-manylinux1_x86_64.whl
    Package Name Version Python Tag ABI Tag Platform Tag

    View full-size slide

  57. useful images
    quay.io/pypa/manylinux1_i686
    quay.io/pypa/manylinux1_x86_64

    View full-size slide

  58. ★ It's an ancient CentOS (for instance

    it has no SNI Support)
    ★ 32bit builds on on 64bit Docker

    typically. Use the linux32 command
    ★ Dockerfile allows you to "cache" steps

    View full-size slide

  59. use std::mem;
    use std::panic;
    fn silent_panic_handler(_pi: &panic::PanicInfo) {
    /* don't do anything here */
    }
    #[no_mangle]
    pub unsafe extern "C" fn mylib_init() {
    panic::set_hook(Box::new(silent_panic_handler));
    }

    View full-size slide

  60. unsafe fn set_err(err: Error, err_out: *mut CError) {
    if err_out.is_null() {
    return;
    }
    let s = format!("{}\x00", err);
    (*err_out).message = Box::into_raw(s.into_boxed_str()) as *mut u8;
    (*err_out).code = err.get_error_code();
    (*err_out).failed = 1;
    }

    View full-size slide

  61. unsafe fn landingpad Result + panic::UnwindSafe, T>(
    f: F, err_out: *mut CError) -> T
    {
    if let Ok(rv) = panic::catch_unwind(f) {
    rv.map_err(|err| set_err(err, err_out)).unwrap_or(mem::zeroed())
    } else {
    set_err(ErrorKind::InternalError.into(), err_out);
    mem::zeroed()
    }
    }

    View full-size slide

  62. macro_rules! export (
    ($n:ident($($an:ident: $aty:ty),*) -> Result<$rv:ty> $body:block) => (
    #[no_mangle]
    pub unsafe extern "C" fn $n($($an: $aty,)* err: *mut CError) -> $rv
    {
    landingpad(|| $body, err)
    }
    );
    );

    View full-size slide

  63. export!(lsm_view_dump_memdb(
    view: *mut View, len_out: *mut c_uint, with_source_contents: c_int,
    with_names: c_int) -> Result<*mut u8>
    {
    let memdb = (*view).dump_memdb(DumpOptions {
    with_source_contents: with_source_contents != 0,
    with_names: with_names != 0,
    })?;
    *len_out = memdb.len() as c_uint;
    Ok(Box::into_raw(memdb.into_boxed_slice()) as *mut u8)
    });

    View full-size slide

  64. typedef void lsm_view_t;
    typedef struct lsm_error_s {
    char *message;
    int failed;
    int code;
    } lsm_error_t;
    char *lsm_view_dump_memdb(const lsm_view_t *view,
    unsigned int *len_out,
    int with_source_contents,
    int with_names,
    lsm_error_t *err);

    View full-size slide

  65. def rustcall(func, *args):
    err = _ffi.new('lsm_error_t *')
    rv = func(*(args + (err,)))
    if not err[0].failed:
    return rv
    try:
    cls = special_errors.get(err[0].code, SourceMapError)
    exc = cls(_ffi.string(err[0].message).decode('utf-8', 'replace'))
    finally:
    _lib.lsm_buffer_free(err[0].message)
    raise exc

    View full-size slide

  66. what is missing

    View full-size slide

  67. better bdist_wheel

    View full-size slide

  68. Reusable Docker Setups

    View full-size slide

  69. RustFFI + Shims

    View full-size slide

  70. (python vs rust source map handling)

    View full-size slide

  71. and when shit
    goes wrong

    View full-size slide

  72. (bug in the binding caused memory leak)

    View full-size slide

  73. it's great, but we
    need better tooling

    View full-size slide