Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Функциональный тулчейн Nix

fwdays
August 14, 2014

Функциональный тулчейн Nix

Владимир Кириллов

fwdays

August 14, 2014
Tweet

More Decks by fwdays

Other Decks in Programming

Transcript

  1. λ-toolchain

    View Slide

  2. λ-toolchain
    @darkproger
    (initially presented on Most Functional Day; Aug 9, 2014)

    View Slide

  3. Nix
    purely functional
    package & configuration mgr
    programming language

    View Slide

  4. Nix
    apt-get / yum
    json + macros
    puppet / chef
    {

    View Slide

  5. Nix
    apt-get / yum
    json + macros
    puppet / chef

    View Slide

  6. the language

    View Slide

  7. View Slide

  8. View Slide

  9. Nix
    (the expression
    language)
    lazy
    expression-oriented
    HOFs, closures

    View Slide

  10. View Slide

  11. % nix-instantiate --eval -E 1
    1

    View Slide

  12. % nix-instantiate --eval -E '
    builtins.trace "hello world" null
    '
    trace: hello world
    null

    View Slide

  13. % nix-instantiate --eval -E '
    [ 1 2 3 ]
    '
    [ 1 2 3 ]
    lists

    View Slide

  14. % nix-instantiate --eval -E '
    { hello = "world"; one = 1; }
    '
    { hello = "world"; one = 1; }
    attribute sets

    View Slide

  15. % nix-instantiate --eval -E '
    rec {
    hello = "world";
    again = hello;
    }
    '
    { again = ; hello = "world"; }
    recursive attrsets

    View Slide

  16. % nix-instantiate --eval --strict -E '
    rec {
    hello = "world";
    again = hello;
    }
    '
    { again = "world"; hello = "world"; }
    (optional)
    strict evaluation

    View Slide

  17. % nix-instantiate --eval -E '
    (a: "hello ${a}") "john doe"
    '
    "hello john doe"
    lambdas

    View Slide

  18. % nix-instantiate --eval -E '
    ((a: b: "hello ${a} ${b}") "john")
    '

    currying

    View Slide

  19. % nix-instantiate --eval -E '
    (((a: b: "hello ${a} ${b}") "john") "doe")
    '
    "hello john doe"
    currying

    View Slide

  20. % nix-repl
    Welcome to Nix version 1.7. Type :? for help.
    nix-repl> :p let
    a = "one"; b = "two";
    f = { foo, bar }: "${foo}: ${bar}”;
    in
    f { foo = a; bar = b; }
    "one: two"
    argument deconstruction

    View Slide

  21. nix-repl> :p let
    foo = “one”; bar = “two";
    f = { foo, bar }: "${foo}: ${bar}”;
    in
    f { inherit foo bar; }
    "one: two"
    scope inheritance

    View Slide

  22. expression files

    View Slide

  23. { lib, ... }: with lib;
    let
    ec2 = {
    "my-elastic-ip" = "54.127.128.129/32";
    };
    rackspace = {
    "huge-box" = "154.24.23.52/32";
    "build-box" = "165.123.32.23/32";
    };
    locations = {
    "home" = "127.0.0.1/32";
    "office" = "169.254.169.254/32";
    };
    in rec {
    netmap = ec2 // rackspace // locations;
    whitelist = attrValues netmap;
    }

    View Slide

  24. { lib, ... }: with lib;
    let
    ec2 = {
    "my-elastic-ip" = "54.127.128.129/32";
    };
    rackspace = {
    "huge-box" = "154.24.23.52/32";
    "build-box" = "165.123.32.23/32";
    };
    locations = {
    "home" = "127.0.0.1/32";
    "office" = "169.254.169.254/32";
    };
    in rec {
    netmap = ec2 // rackspace // locations;
    whitelist = attrValues netmap;
    }

    View Slide

  25. % nix-instantiate --eval my-networks.nix

    View Slide

  26. % nix-instantiate --eval my-networks.nix

    { lib, ... }: with lib;

    View Slide

  27. % nix-instantiate --eval \
    --arg lib '(import {}).lib' \
    my-networks.nix
    { netmap = ; whitelist = ; }

    View Slide

  28. % nix-instantiate --eval --strict \
    --arg lib '(import {}).lib' \
    my-networks.nix
    { netmap = { build-box = "165.123.32.23/32"; home =
    "127.0.0.1/32"; huge-box = "154.24.23.52/32"; my-elastic-ip
    = "54.127.128.129/32"; office = "169.254.169.254/32"; };
    whitelist = [ "165.123.32.23/32" "127.0.0.1/32"
    "154.24.23.52/32" "54.127.128.129/32"
    "169.254.169.254/32" ]; }

    View Slide

  29. % nix-instantiate --eval --strict -E '
    builtins.toJSON import ./my-networks.nix
    { inherit (import {}) lib; }
    '
    error: cannot convert a partially applied built-
    in function to JSON

    View Slide

  30. % nix-instantiate --eval --strict -E '
    builtins.toJSON
    (import ./my-networks.nix { inherit (import
    {}) lib; })
    ' | jq -r . | jq .
    {
    "whitelist": [
    "165.123.32.23/32",
    "127.0.0.1/32",
    "154.24.23.52/32",
    "54.127.128.129/32",
    "169.254.169.254/32"
    ],
    "netmap": {
    "office": "169.254.169.254/32",
    "my-elastic-ip": "54.127.128.129/32",
    "huge-box": "154.24.23.52/32",
    "home": "127.0.0.1/32",
    "build-box": "165.123.32.23/32"
    }
    }

    View Slide

  31. ''
    #!${pkgs.bash}/bin/bash
    # this string is going to be indented
    cd ${if cfg.dataDir != ""
    then cfg.dataDir
    else "/var/empty"}
    exec ${cfg.command}
    '';
    smart interpolation

    View Slide

  32. ''
    #!${pkgs.bash}/bin/bash
    # this string is going to be indented
    cd ${if cfg.dataDir != ""
    then cfg.dataDir
    else "/var/empty"}
    exec ${cfg.command}
    '';
    "antiquotation"

    View Slide

  33. derivations

    View Slide

  34. let
    system = builtins.currentSystem;
    builder = /bin/bash;
    in rec {
    writeText = name: text: derivation {
    inherit system builder name;
    args = ["-c" "echo -n '${text}' > $out"];
    };
    }

    View Slide

  35. % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv

    View Slide

  36. % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    the store

    View Slide

  37. % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    % nix-instantiate -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    warning: you did not specify `--add-root'; the result
    might be removed by the garbage collector
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv

    View Slide

  38. Derive(
    [("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello","","")],
    [],
    ["/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"],
    "x86_64-darwin",
    "/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash",
    ["-c","echo -n 'world' > $out"],
    [("builder","/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"),
    ("name","hello"),
    ("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello"),
    ("system","x86_64-darwin")]
    )
    "annotated term"

    View Slide

  39. % nix-store --realise /nix/store/
    ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    these derivations will be built:
    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    building path(s) `/nix/store/
    p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello'
    warning: you did not specify `--add-root'; the result might
    be removed by the garbage collector
    /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    realising the derivation

    View Slide

  40. % nix-store --realise /nix/store/
    ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    warning: you did not specify `--add-root'; the result might be
    removed by the garbage collector
    /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    ... only once

    View Slide

  41. % nix-store --print-env /nix/store/
    ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
    export builder; builder='/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash'
    export name; name='hello'
    export out; out='/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello'
    export system; system='x86_64-darwin'
    export _args; _args='-c echo -n '\''world'\'' > $out'
    reproducible env

    View Slide

  42. % nix-shell --pure -E '
    (import ./ourlib.nix).writeText "hello" "world"
    '
    [nix-shell:~]$
    nix-shell

    View Slide

  43. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    world

    View Slide

  44. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello
    world
    hash of all inputs

    View Slide

  45. derivations package manager

    View Slide

  46. View Slide

  47. % ll $(nix-instantiate --eval -E '')/
    total 24
    -rw-r--r-- 1 proger staff 1.6K Jun 17 13:36 COPYING
    -rw-r--r-- 1 proger staff 677B Jun 23 17:17 README.md
    -rw-r--r-- 1 proger staff 213B Jun 17 13:36 default.nix
    drwxr-xr-x 15 proger staff 510B Jul 21 10:17 doc/
    drwxr-xr-x 22 proger staff 748B Jul 21 10:17 lib/
    drwxr-xr-x 3 proger staff 102B Jun 17 13:36 maintainers/
    drwxr-xr-x 13 proger staff 442B Jul 21 10:17 nixos/
    drwxr-xr-x 16 proger staff 544B Jun 17 13:36 pkgs/

    View Slide

  48. % cat ~dev/nix/nixpkgs/pkgs/os-specific/linux/latencytop/default.nix
    { stdenv, fetchurl, ncurses, glib, pkgconfig, gtk }:
    stdenv.mkDerivation rec {
    name = "latencytop-0.5";
    patchPhase = "sed -i s,/usr,$out, Makefile";
    preInstall = "mkdir -p $out/sbin";
    src = fetchurl {
    urls = [ "http://latencytop.org/download/${name}.tar.gz"
    "http://dbg.download.sourcemage.org/mirror/latencytop-0.5.tar.gz" ];
    sha256 = "1vq3j9zdab6njly2wp900b3d5244mnxfm88j2bkiinbvxbxp4zwy";
    };
    buildInputs = [ ncurses glib pkgconfig gtk ];
    meta = {
    homepage = http://latencytop.org;
    description = "Tool to show kernel reports on latencies (LATENCYTOP option)";
    license = "GPLv2";
    maintainers = [ stdenv.lib.maintainers.viric ];
    platforms = stdenv.lib.platforms.linux;
    };
    }

    View Slide

  49. stdenv
    basic toolchain for C
    ./configure --prefix=$out
    make
    make install (and more)
    unix idiosyncrasies
    unix flavours
    monkey-patching (patchelf)
    nix-shell matters even more

    View Slide

  50. stdenv for
    $LANGUAGE
    build on top of language-
    specific package manager
    integrate with the rest of the
    world

    View Slide

  51. % nix-store -qR /nix/store/
    lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82
    /nix/store/x52mfsvjaz1k82iy8hh5bmh1jhz62kwg-gmp-5.1.3
    /nix/store/q4dp57r561pqps48sqnps74xwz9fg9b4-isl-0.11.1
    /nix/store/939lir5rs9axn2pch7sxgxwpvk2yq20d-cloog-0.18.0
    /nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8
    /nix/store/j81x6cd2zdqp1f0yprghgd266d5zgjx0-mpfr-3.1.2
    /nix/store/jqmfwcky6r6m177ak87rsrfg825z383w-zlib-1.2.8
    /nix/store/m551ya45l9gg0wclllhniszrccl3szb6-mpc-1.0.1
    /nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2
    /nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3
    /nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82
    store closure

    View Slide

  52. % nix-copy-closure --to anotherbox /nix/store/
    lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82
    copying 4 missing paths to ‘anotherbox’...
    importing path `/nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8'
    load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s
    load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s
    importing path `/nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2'
    importing path `/nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3'
    importing path `/nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82'

    View Slide

  53. More distributed builds (fanout through ssh)
    caching for free

    View Slide

  54. View Slide

  55. NixOS
    binary channels
    user environments (profiles)
    /etc/nixos/configuration.nix
    generations / rollbacks
    Solaris-like "boot configurations"
    mostly designed for laptops /
    legacy systems :(

    View Slide

  56. configuration
    management

    View Slide

  57. misconfiguration
    management

    View Slide

  58. configuration
    mismanagement

    View Slide

  59. puppet
    chef / ansible / salt / whatever

    View Slide

  60. View Slide

  61. View Slide

  62. let
    region = "ap-southeast-1";
    accessKeyId = "devops";
    gen-instance = { resources, ... }: (import ./thumbor.nix {}) {
    deployment.ec2 = {
    inherit accessKeyId region;
    instanceType = "c3.2xlarge";
    };
    fileSystems."/data" = {
    ec2.disk = "ephemeral0";
    };
    };
    in
    {
    resources.ec2KeyPairs.default = {
    inherit accessKeyId region;
    };
    thumbor1 = gen-instance;
    thumbor2 = gen-instance;
    thumbor3 = gen-instance;
    thumbor4 = gen-instance;
    }

    View Slide

  63. ????

    View Slide

  64. View Slide

  65. { lib }: with lib;
    rec {
    mapcat = f: x: concatLists (mapAttrsToList f x);
    mapcata = f: x: listToAttrs (mapcat f x);
    }
    maps/filters included

    View Slide

  66. resources.elbs.web = { resources, config, pkgs, lib, ... }: with lib; {
    inherit region accessKeyId;
    name = "${name}-web";
    subnets = [ resources.subnets.default ];
    securityGroups = [ resources.ec2SecurityGroups.default ];
    machines = mapAttrsToList (name: m: m) resources.machines;
    route53Aliases = mapcata (_name: app: (map (prefix: {
    name = prefix + app.dns.public.name;
    value = {
    zoneId = app.dns.public.zoneId;
    };
    }) ["" "*."])) all-apps;
    };

    View Slide

  67. Integration
    --eval --strict
    --xml
    builtins.toJSON
    builtins.fromJSON
    staged evaluations
    perl bindings (omg please don't)

    View Slide

  68. Future hnix
    https:/
    /github.com/jwiegley/hnix

    View Slide

  69. More
    nix-docker
    cabal2nix
    python2nix
    hydra (Nix CI)
    guix
    smart community!

    View Slide

  70. Where it helps
    (aka show me the money)

    View Slide

  71. reconciling configuration hell
    (tooling helps)

    View Slide

  72. separating implementation and
    declaration while keeping code
    and configuration together

    View Slide

  73. managing dependency hell
    (e.g. sudden rewrites of
    services from Go to Python)

    View Slide

  74. bridging multilang dependencies
    (i.e. Python with C with Haskell)

    View Slide

  75. writing shell scripts!

    View Slide

  76. VPC IP<->Route53
    mapping
    https://gist.github.com/proger/49e7ae550f4e2d4c99af

    View Slide

  77. zone=ZOZONEZONEZONE
    date=$(curl -s -I https://route53.amazonaws.com/date \
    | awk '/^Date: / {
    sub("Date: ", "", $0);
    sub("\\r", "", $0); print $0 }')
    set -- $(curl -s http://169.254.169.254/latest/meta-data/
    iam/security-credentials/dns \
    | jq -r '.SecretAccessKey, .AccessKeyId, .Token')
    signature=$(echo -n $date \
    | openssl dgst -binary -sha1 -hmac $1 \
    | base64)
    auth_header="X-Amzn-Authorization: AWS3-HTTPS
    AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature"
    hostname=$(hostname).doge-networking-enterprises.com
    local_hostname=$(curl -s http://169.254.169.254/latest/
    meta-data/local-hostname)

    View Slide

  78. zone
    date
    | awk
    sub("Date: ", "", $0);
    sub("\\r", "", $0); print $0 }'
    set
    iam/security-credentials/dns
    | jq -r
    signature
    | openssl dgst -binary -sha1 -hmac
    | base64
    auth_header
    AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature"
    hostname
    local_hostname
    meta-data/local-hostname
    curl
    awk
    curl
    jq
    openssl
    coreutils
    curl
    bash

    View Slide

  79. bash = "${pkgs.bash}/bin/bash";
    base64 = "${pkgs.coreutils}/bin/base64";
    jq = "${pkgs.jq}/bin/jq";
    curl = "${pkgs.curl}/bin/curl -s --retry --fail";
    awk = "${pkgs.gawk}/bin/awk";
    openssl = "${pkgs.openssl}/bin/openssl";

    View Slide

  80. zone
    date
    | awk
    sub("Date: ", "", $0);
    sub("\\r", "", $0); print $0 }'
    set
    iam/security-credentials/dns
    | jq -r
    signature
    | openssl dgst -binary -sha1 -hmac
    | base64
    auth_header
    AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature"
    hostname
    local_hostname
    meta-data/local-hostname
    ${curl}
    ${awk}
    ${jq}
    ${openssl}
    ${base64}
    ${curl}
    ${curl}
    ${bash}

    View Slide

  81. speeding up Haskell development
    (benefits of cabal sandbox
    without its disadvantages)

    View Slide

  82. managing Cabal hell &
    GHC versions

    View Slide

  83. library isolation for C
    and C++ software
    (that's usually where containers shine,
    but works for us)

    View Slide

  84. teamwork & pull requests

    View Slide

  85. developer environments
    close to production

    View Slide

  86. lifting your legacy apps into
    the cloud
    (your apps don't need configuration,
    remember?)

    View Slide

  87. Getting
    started
    mix and match
    Nix with Docker
    Nix on Debian with Chef
    Nix on Debian with NixOps

    View Slide

  88. https:/
    /github.com/zalora
    we're using and improving Nix

    View Slide

  89. stay tuned
    ask me anything
    @darkproger

    View Slide