Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Функциональный тулчейн Nix

fwdays
August 14, 2014

Функциональный тулчейн Nix

Владимир Кириллов

fwdays

August 14, 2014
Tweet

More Decks by fwdays

Other Decks in Programming

Transcript

  1. λ-toolchain

    View full-size slide

  2. λ-toolchain
    @darkproger
    (initially presented on Most Functional Day; Aug 9, 2014)

    View full-size slide

  3. Nix
    purely functional
    package & configuration mgr
    programming language

    View full-size slide

  4. Nix
    apt-get / yum
    json + macros
    puppet / chef
    {

    View full-size slide

  5. Nix
    apt-get / yum
    json + macros
    puppet / chef

    View full-size slide

  6. the language

    View full-size slide

  7. Nix
    (the expression
    language)
    lazy
    expression-oriented
    HOFs, closures

    View full-size slide

  8. % nix-instantiate --eval -E 1
    1

    View full-size slide

  9. % nix-instantiate --eval -E '
    builtins.trace "hello world" null
    '
    trace: hello world
    null

    View full-size slide

  10. % nix-instantiate --eval -E '
    [ 1 2 3 ]
    '
    [ 1 2 3 ]
    lists

    View full-size slide

  11. % nix-instantiate --eval -E '
    { hello = "world"; one = 1; }
    '
    { hello = "world"; one = 1; }
    attribute sets

    View full-size slide

  12. % nix-instantiate --eval -E '
    rec {
    hello = "world";
    again = hello;
    }
    '
    { again = ; hello = "world"; }
    recursive attrsets

    View full-size slide

  13. % nix-instantiate --eval --strict -E '
    rec {
    hello = "world";
    again = hello;
    }
    '
    { again = "world"; hello = "world"; }
    (optional)
    strict evaluation

    View full-size slide

  14. % nix-instantiate --eval -E '
    (a: "hello ${a}") "john doe"
    '
    "hello john doe"
    lambdas

    View full-size slide

  15. % nix-instantiate --eval -E '
    ((a: b: "hello ${a} ${b}") "john")
    '

    currying

    View full-size slide

  16. % nix-instantiate --eval -E '
    (((a: b: "hello ${a} ${b}") "john") "doe")
    '
    "hello john doe"
    currying

    View full-size slide

  17. % nix-repl
    Welcome to Nix version 1.7. Type :? for help.
    nix-repl> :p let
    a = "one"; b = "two";
    f = { foo, bar }: "${foo}: ${bar}”;
    in
    f { foo = a; bar = b; }
    "one: two"
    argument deconstruction

    View full-size slide

  18. nix-repl> :p let
    foo = “one”; bar = “two";
    f = { foo, bar }: "${foo}: ${bar}”;
    in
    f { inherit foo bar; }
    "one: two"
    scope inheritance

    View full-size slide

  19. expression files

    View full-size slide

  20. { lib, ... }: with lib;
    let
    ec2 = {
    "my-elastic-ip" = "54.127.128.129/32";
    };
    rackspace = {
    "huge-box" = "154.24.23.52/32";
    "build-box" = "165.123.32.23/32";
    };
    locations = {
    "home" = "127.0.0.1/32";
    "office" = "169.254.169.254/32";
    };
    in rec {
    netmap = ec2 // rackspace // locations;
    whitelist = attrValues netmap;
    }

    View full-size slide

  21. { lib, ... }: with lib;
    let
    ec2 = {
    "my-elastic-ip" = "54.127.128.129/32";
    };
    rackspace = {
    "huge-box" = "154.24.23.52/32";
    "build-box" = "165.123.32.23/32";
    };
    locations = {
    "home" = "127.0.0.1/32";
    "office" = "169.254.169.254/32";
    };
    in rec {
    netmap = ec2 // rackspace // locations;
    whitelist = attrValues netmap;
    }

    View full-size slide

  22. % nix-instantiate --eval my-networks.nix

    View full-size slide

  23. % nix-instantiate --eval my-networks.nix

    { lib, ... }: with lib;

    View full-size slide

  24. % nix-instantiate --eval \
    --arg lib '(import {}).lib' \
    my-networks.nix
    { netmap = ; whitelist = ; }

    View full-size slide

  25. % nix-instantiate --eval --strict \
    --arg lib '(import {}).lib' \
    my-networks.nix
    { netmap = { build-box = "165.123.32.23/32"; home =
    "127.0.0.1/32"; huge-box = "154.24.23.52/32"; my-elastic-ip
    = "54.127.128.129/32"; office = "169.254.169.254/32"; };
    whitelist = [ "165.123.32.23/32" "127.0.0.1/32"
    "154.24.23.52/32" "54.127.128.129/32"
    "169.254.169.254/32" ]; }

    View full-size slide

  26. % nix-instantiate --eval --strict -E '
    builtins.toJSON import ./my-networks.nix
    { inherit (import {}) lib; }
    '
    error: cannot convert a partially applied built-
    in function to JSON

    View full-size slide

  27. % nix-instantiate --eval --strict -E '
    builtins.toJSON
    (import ./my-networks.nix { inherit (import
    {}) lib; })
    ' | jq -r . | jq .
    {
    "whitelist": [
    "165.123.32.23/32",
    "127.0.0.1/32",
    "154.24.23.52/32",
    "54.127.128.129/32",
    "169.254.169.254/32"
    ],
    "netmap": {
    "office": "169.254.169.254/32",
    "my-elastic-ip": "54.127.128.129/32",
    "huge-box": "154.24.23.52/32",
    "home": "127.0.0.1/32",
    "build-box": "165.123.32.23/32"
    }
    }

    View full-size slide

  28. ''
    #!${pkgs.bash}/bin/bash
    # this string is going to be indented
    cd ${if cfg.dataDir != ""
    then cfg.dataDir
    else "/var/empty"}
    exec ${cfg.command}
    '';
    smart interpolation

    View full-size slide

  29. ''
    #!${pkgs.bash}/bin/bash
    # this string is going to be indented
    cd ${if cfg.dataDir != ""
    then cfg.dataDir
    else "/var/empty"}
    exec ${cfg.command}
    '';
    "antiquotation"

    View full-size slide

  30. let
    system = builtins.currentSystem;
    builder = /bin/bash;
    in rec {
    writeText = name: text: derivation {
    inherit system builder name;
    args = ["-c" "echo -n '${text}' > $out"];
    };
    }

    View full-size slide

  31. % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv

    View full-size slide

  32. % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    the store

    View full-size slide

  33. % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv

    View full-size slide

  34. Derive(
    [("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello","","")],
    [],
    ["/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"],
    "x86_64-darwin",
    "/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash",
    ["-c","echo -n 'world' > $out"],
    [("builder","/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"),
    ("name","hello"),
    ("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello"),
    ("system","x86_64-darwin")]
    )
    "annotated term"

    View full-size slide

  35. % nix-store --realise /nix/store/
    ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    these derivations will be built:
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    building path(s) `/nix/store/
    p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello'
    warning: you did not specify `--add-root'; the result might
    be removed by the garbage collector
    /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    realising the derivation

    View full-size slide

  36. % nix-store --realise /nix/store/
    ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    warning: you did not specify `--add-root'; the result might be
    removed by the garbage collector
    /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    ... only once

    View full-size slide

  37. % nix-store --print-env /nix/store/
    ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    export builder; builder='/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash'
    export name; name='hello'
    export out; out='/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello'
    export system; system='x86_64-darwin'
    export _args; _args='-c echo -n '\''world'\'' > $out'
    reproducible env

    View full-size slide

  38. % nix-shell --pure -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    [nix-shell:~]$
    nix-shell

    View full-size slide

  39. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    world

    View full-size slide

  40. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    world
    hash of all inputs

    View full-size slide

  41. derivations package manager

    View full-size slide

  42. % ll $(nix-instantiate --eval -E '')/
    total 24
    -rw-r--r-- 1 proger staff 1.6K Jun 17 13:36 COPYING
    -rw-r--r-- 1 proger staff 677B Jun 23 17:17 README.md
    -rw-r--r-- 1 proger staff 213B Jun 17 13:36 default.nix
    drwxr-xr-x 15 proger staff 510B Jul 21 10:17 doc/
    drwxr-xr-x 22 proger staff 748B Jul 21 10:17 lib/
    drwxr-xr-x 3 proger staff 102B Jun 17 13:36 maintainers/
    drwxr-xr-x 13 proger staff 442B Jul 21 10:17 nixos/
    drwxr-xr-x 16 proger staff 544B Jun 17 13:36 pkgs/

    View full-size slide

  43. % cat ~dev/nix/nixpkgs/pkgs/os-specific/linux/latencytop/default.nix
    { stdenv, fetchurl, ncurses, glib, pkgconfig, gtk }:
    stdenv.mkDerivation rec {
    name = "latencytop-0.5";
    patchPhase = "sed -i s,/usr,$out, Makefile";
    preInstall = "mkdir -p $out/sbin";
    src = fetchurl {
    urls = [ "http://latencytop.org/download/${name}.tar.gz"
    "http://dbg.download.sourcemage.org/mirror/latencytop-0.5.tar.gz" ];
    sha256 = "1vq3j9zdab6njly2wp900b3d5244mnxfm88j2bkiinbvxbxp4zwy";
    };
    buildInputs = [ ncurses glib pkgconfig gtk ];
    meta = {
    homepage = http://latencytop.org;
    description = "Tool to show kernel reports on latencies (LATENCYTOP option)";
    license = "GPLv2";
    maintainers = [ stdenv.lib.maintainers.viric ];
    platforms = stdenv.lib.platforms.linux;
    };
    }

    View full-size slide

  44. stdenv
    basic toolchain for C
    ./configure --prefix=$out
    make
    make install (and more)
    unix idiosyncrasies
    unix flavours
    monkey-patching (patchelf)
    nix-shell matters even more

    View full-size slide

  45. stdenv for
    $LANGUAGE
    build on top of language-
    specific package manager
    integrate with the rest of the
    world

    View full-size slide

  46. % nix-store -qR /nix/store/
    lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82
    /nix/store/x52mfsvjaz1k82iy8hh5bmh1jhz62kwg-gmp-5.1.3
    /nix/store/q4dp57r561pqps48sqnps74xwz9fg9b4-isl-0.11.1
    /nix/store/939lir5rs9axn2pch7sxgxwpvk2yq20d-cloog-0.18.0
    /nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8
    /nix/store/j81x6cd2zdqp1f0yprghgd266d5zgjx0-mpfr-3.1.2
    /nix/store/jqmfwcky6r6m177ak87rsrfg825z383w-zlib-1.2.8
    /nix/store/m551ya45l9gg0wclllhniszrccl3szb6-mpc-1.0.1
    /nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2
    /nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3
    /nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82
    store closure

    View full-size slide

  47. % nix-copy-closure --to anotherbox /nix/store/
    lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82
    copying 4 missing paths to ‘anotherbox’...
    importing path `/nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8'
    load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s
    load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s
    importing path `/nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2'
    importing path `/nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3'
    importing path `/nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82'

    View full-size slide

  48. More distributed builds (fanout through ssh)
    caching for free

    View full-size slide

  49. NixOS
    binary channels
    user environments (profiles)
    /etc/nixos/configuration.nix
    generations / rollbacks
    Solaris-like "boot configurations"
    mostly designed for laptops /
    legacy systems :(

    View full-size slide

  50. configuration
    management

    View full-size slide

  51. misconfiguration
    management

    View full-size slide

  52. configuration
    mismanagement

    View full-size slide

  53. puppet
    chef / ansible / salt / whatever

    View full-size slide

  54. let
    region = "ap-southeast-1";
    accessKeyId = "devops";
    gen-instance = { resources, ... }: (import ./thumbor.nix {}) {
    deployment.ec2 = {
    inherit accessKeyId region;
    instanceType = "c3.2xlarge";
    };
    fileSystems."/data" = {
    ec2.disk = "ephemeral0";
    };
    };
    in
    {
    resources.ec2KeyPairs.default = {
    inherit accessKeyId region;
    };
    thumbor1 = gen-instance;
    thumbor2 = gen-instance;
    thumbor3 = gen-instance;
    thumbor4 = gen-instance;
    }

    View full-size slide

  55. { lib }: with lib;
    rec {
    mapcat = f: x: concatLists (mapAttrsToList f x);
    mapcata = f: x: listToAttrs (mapcat f x);
    }
    maps/filters included

    View full-size slide

  56. resources.elbs.web = { resources, config, pkgs, lib, ... }: with lib; {
    inherit region accessKeyId;
    name = "${name}-web";
    subnets = [ resources.subnets.default ];
    securityGroups = [ resources.ec2SecurityGroups.default ];
    machines = mapAttrsToList (name: m: m) resources.machines;
    route53Aliases = mapcata (_name: app: (map (prefix: {
    name = prefix + app.dns.public.name;
    value = {
    zoneId = app.dns.public.zoneId;
    };
    }) ["" "*."])) all-apps;
    };

    View full-size slide

  57. Integration
    --eval --strict
    --xml
    builtins.toJSON
    builtins.fromJSON
    staged evaluations
    perl bindings (omg please don't)

    View full-size slide

  58. Future hnix
    https:/
    /github.com/jwiegley/hnix

    View full-size slide

  59. More
    nix-docker
    cabal2nix
    python2nix
    hydra (Nix CI)
    guix
    smart community!

    View full-size slide

  60. Where it helps
    (aka show me the money)

    View full-size slide

  61. reconciling configuration hell
    (tooling helps)

    View full-size slide

  62. separating implementation and
    declaration while keeping code
    and configuration together

    View full-size slide

  63. managing dependency hell
    (e.g. sudden rewrites of
    services from Go to Python)

    View full-size slide

  64. bridging multilang dependencies
    (i.e. Python with C with Haskell)

    View full-size slide

  65. writing shell scripts!

    View full-size slide

  66. VPC IP<->Route53
    mapping
    https://gist.github.com/proger/49e7ae550f4e2d4c99af

    View full-size slide

  67. zone=ZOZONEZONEZONE
    date=$(curl -s -I https://route53.amazonaws.com/date \
    | awk '/^Date: / {
    sub("Date: ", "", $0);
    sub("\\r", "", $0); print $0 }')
    set -- $(curl -s http://169.254.169.254/latest/meta-data/
    iam/security-credentials/dns \
    | jq -r '.SecretAccessKey, .AccessKeyId, .Token')
    signature=$(echo -n $date \
    | openssl dgst -binary -sha1 -hmac $1 \
    | base64)
    auth_header="X-Amzn-Authorization: AWS3-HTTPS
    AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature"
    hostname=$(hostname).doge-networking-enterprises.com
    local_hostname=$(curl -s http://169.254.169.254/latest/
    meta-data/local-hostname)

    View full-size slide

  68. zone
    date
    | awk
    sub("Date: ", "", $0);
    sub("\\r", "", $0); print $0 }'
    set
    iam/security-credentials/dns
    | jq -r
    signature
    | openssl dgst -binary -sha1 -hmac
    | base64
    auth_header
    AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature"
    hostname
    local_hostname
    meta-data/local-hostname
    curl
    awk
    curl
    jq
    openssl
    coreutils
    curl
    bash

    View full-size slide

  69. bash = "${pkgs.bash}/bin/bash";
    base64 = "${pkgs.coreutils}/bin/base64";
    jq = "${pkgs.jq}/bin/jq";
    curl = "${pkgs.curl}/bin/curl -s --retry --fail";
    awk = "${pkgs.gawk}/bin/awk";
    openssl = "${pkgs.openssl}/bin/openssl";

    View full-size slide

  70. zone
    date
    | awk
    sub("Date: ", "", $0);
    sub("\\r", "", $0); print $0 }'
    set
    iam/security-credentials/dns
    | jq -r
    signature
    | openssl dgst -binary -sha1 -hmac
    | base64
    auth_header
    AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature"
    hostname
    local_hostname
    meta-data/local-hostname
    ${curl}
    ${awk}
    ${jq}
    ${openssl}
    ${base64}
    ${curl}
    ${curl}
    ${bash}

    View full-size slide

  71. speeding up Haskell development
    (benefits of cabal sandbox
    without its disadvantages)

    View full-size slide

  72. managing Cabal hell &
    GHC versions

    View full-size slide

  73. library isolation for C
    and C++ software
    (that's usually where containers shine,
    but works for us)

    View full-size slide

  74. teamwork & pull requests

    View full-size slide

  75. developer environments
    close to production

    View full-size slide

  76. lifting your legacy apps into
    the cloud
    (your apps don't need configuration,
    remember?)

    View full-size slide

  77. Getting
    started
    mix and match
    Nix with Docker
    Nix on Debian with Chef
    Nix on Debian with NixOps

    View full-size slide

  78. https:/
    /github.com/zalora
    we're using and improving Nix

    View full-size slide

  79. stay tuned
    ask me anything
    @darkproger

    View full-size slide