Функциональный тулчейн Nix

Dd3f18c87b851137000c7427d7bd5d32?s=47 fwdays
August 14, 2014

Функциональный тулчейн Nix

Владимир Кириллов

Dd3f18c87b851137000c7427d7bd5d32?s=128

fwdays

August 14, 2014
Tweet

Transcript

  1. λ-toolchain

  2. λ-toolchain @darkproger (initially presented on Most Functional Day; Aug 9,

    2014)
  3. Nix purely functional package & configuration mgr programming language

  4. Nix apt-get / yum json + macros puppet / chef

    {
  5. Nix apt-get / yum json + macros puppet / chef

  6. the language

  7. None
  8. None
  9. Nix (the expression language) lazy expression-oriented HOFs, closures

  10. None
  11. % nix-instantiate --eval -E 1 1

  12. % nix-instantiate --eval -E ' builtins.trace "hello world" null '

    trace: hello world null
  13. % nix-instantiate --eval -E ' [ 1 2 3 ]

    ' [ 1 2 3 ] lists
  14. % nix-instantiate --eval -E ' { hello = "world"; one

    = 1; } ' { hello = "world"; one = 1; } attribute sets
  15. % nix-instantiate --eval -E ' rec { hello = "world";

    again = hello; } ' { again = <CODE>; hello = "world"; } recursive attrsets
  16. % nix-instantiate --eval --strict -E ' rec { hello =

    "world"; again = hello; } ' { again = "world"; hello = "world"; } (optional) strict evaluation
  17. % nix-instantiate --eval -E ' (a: "hello ${a}") "john doe"

    ' "hello john doe" lambdas
  18. % nix-instantiate --eval -E ' ((a: b: "hello ${a} ${b}")

    "john") ' <LAMBDA> currying
  19. % nix-instantiate --eval -E ' (((a: b: "hello ${a} ${b}")

    "john") "doe") ' "hello john doe" currying
  20. % nix-repl Welcome to Nix version 1.7. Type :? for

    help. nix-repl> :p let a = "one"; b = "two"; f = { foo, bar }: "${foo}: ${bar}”; in f { foo = a; bar = b; } "one: two" argument deconstruction
  21. nix-repl> :p let foo = “one”; bar = “two"; f

    = { foo, bar }: "${foo}: ${bar}”; in f { inherit foo bar; } "one: two" scope inheritance
  22. expression files

  23. { lib, ... }: with lib; let ec2 = {

    "my-elastic-ip" = "54.127.128.129/32"; }; rackspace = { "huge-box" = "154.24.23.52/32"; "build-box" = "165.123.32.23/32"; }; locations = { "home" = "127.0.0.1/32"; "office" = "169.254.169.254/32"; }; in rec { netmap = ec2 // rackspace // locations; whitelist = attrValues netmap; }
  24. { lib, ... }: with lib; let ec2 = {

    "my-elastic-ip" = "54.127.128.129/32"; }; rackspace = { "huge-box" = "154.24.23.52/32"; "build-box" = "165.123.32.23/32"; }; locations = { "home" = "127.0.0.1/32"; "office" = "169.254.169.254/32"; }; in rec { netmap = ec2 // rackspace // locations; whitelist = attrValues netmap; }
  25. % nix-instantiate --eval my-networks.nix <LAMBDA>

  26. % nix-instantiate --eval my-networks.nix <LAMBDA> { lib, ... }: with

    lib;
  27. % nix-instantiate --eval \ --arg lib '(import <nixpkgs> {}).lib' \

    my-networks.nix { netmap = <CODE>; whitelist = <CODE>; }
  28. % nix-instantiate --eval --strict \ --arg lib '(import <nixpkgs> {}).lib'

    \ my-networks.nix { netmap = { build-box = "165.123.32.23/32"; home = "127.0.0.1/32"; huge-box = "154.24.23.52/32"; my-elastic-ip = "54.127.128.129/32"; office = "169.254.169.254/32"; }; whitelist = [ "165.123.32.23/32" "127.0.0.1/32" "154.24.23.52/32" "54.127.128.129/32" "169.254.169.254/32" ]; }
  29. % nix-instantiate --eval --strict -E ' builtins.toJSON import ./my-networks.nix {

    inherit (import <nixpkgs> {}) lib; } ' error: cannot convert a partially applied built- in function to JSON
  30. % nix-instantiate --eval --strict -E ' builtins.toJSON (import ./my-networks.nix {

    inherit (import <nixpkgs> {}) lib; }) ' | jq -r . | jq . { "whitelist": [ "165.123.32.23/32", "127.0.0.1/32", "154.24.23.52/32", "54.127.128.129/32", "169.254.169.254/32" ], "netmap": { "office": "169.254.169.254/32", "my-elastic-ip": "54.127.128.129/32", "huge-box": "154.24.23.52/32", "home": "127.0.0.1/32", "build-box": "165.123.32.23/32" } }
  31. '' #!${pkgs.bash}/bin/bash # this string is going to be indented

    cd ${if cfg.dataDir != "" then cfg.dataDir else "/var/empty"} exec ${cfg.command} ''; smart interpolation
  32. '' #!${pkgs.bash}/bin/bash # this string is going to be indented

    cd ${if cfg.dataDir != "" then cfg.dataDir else "/var/empty"} exec ${cfg.command} ''; "antiquotation"
  33. derivations

  34. let system = builtins.currentSystem; builder = /bin/bash; in rec {

    writeText = name: text: derivation { inherit system builder name; args = ["-c" "echo -n '${text}' > $out"]; }; }
  35. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning:

    you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
  36. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning:

    you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv the store
  37. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning:

    you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
  38. Derive( [("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello","","")], [], ["/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"], "x86_64-darwin", "/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash", ["-c","echo -n 'world' >

    $out"], [("builder","/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"), ("name","hello"), ("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello"), ("system","x86_64-darwin")] ) "annotated term"
  39. % nix-store --realise /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv these derivations will be built:

    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv building path(s) `/nix/store/ p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello realising the derivation
  40. % nix-store --realise /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv warning: you did not specify

    `--add-root'; the result might be removed by the garbage collector /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello ... only once
  41. % nix-store --print-env /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv export builder; builder='/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash' export name;

    name='hello' export out; out='/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello' export system; system='x86_64-darwin' export _args; _args='-c echo -n '\''world'\'' > $out' reproducible env
  42. % nix-shell --pure -E ' (import ./ourlib.nix).writeText "hello" "world" '

    [nix-shell:~]$ nix-shell
  43. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello world

  44. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello world hash of all inputs

  45. derivations package manager

  46. None
  47. % ll $(nix-instantiate --eval -E '<nixpkgs>')/ total 24 -rw-r--r-- 1

    proger staff 1.6K Jun 17 13:36 COPYING -rw-r--r-- 1 proger staff 677B Jun 23 17:17 README.md -rw-r--r-- 1 proger staff 213B Jun 17 13:36 default.nix drwxr-xr-x 15 proger staff 510B Jul 21 10:17 doc/ drwxr-xr-x 22 proger staff 748B Jul 21 10:17 lib/ drwxr-xr-x 3 proger staff 102B Jun 17 13:36 maintainers/ drwxr-xr-x 13 proger staff 442B Jul 21 10:17 nixos/ drwxr-xr-x 16 proger staff 544B Jun 17 13:36 pkgs/
  48. % cat ~dev/nix/nixpkgs/pkgs/os-specific/linux/latencytop/default.nix { stdenv, fetchurl, ncurses, glib, pkgconfig, gtk

    }: stdenv.mkDerivation rec { name = "latencytop-0.5"; patchPhase = "sed -i s,/usr,$out, Makefile"; preInstall = "mkdir -p $out/sbin"; src = fetchurl { urls = [ "http://latencytop.org/download/${name}.tar.gz" "http://dbg.download.sourcemage.org/mirror/latencytop-0.5.tar.gz" ]; sha256 = "1vq3j9zdab6njly2wp900b3d5244mnxfm88j2bkiinbvxbxp4zwy"; }; buildInputs = [ ncurses glib pkgconfig gtk ]; meta = { homepage = http://latencytop.org; description = "Tool to show kernel reports on latencies (LATENCYTOP option)"; license = "GPLv2"; maintainers = [ stdenv.lib.maintainers.viric ]; platforms = stdenv.lib.platforms.linux; }; }
  49. stdenv basic toolchain for C ./configure --prefix=$out make make install

    (and more) unix idiosyncrasies unix flavours monkey-patching (patchelf) nix-shell matters even more
  50. stdenv for $LANGUAGE build on top of language- specific package

    manager integrate with the rest of the world
  51. % nix-store -qR /nix/store/ lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 /nix/store/x52mfsvjaz1k82iy8hh5bmh1jhz62kwg-gmp-5.1.3 /nix/store/q4dp57r561pqps48sqnps74xwz9fg9b4-isl-0.11.1 /nix/store/939lir5rs9axn2pch7sxgxwpvk2yq20d-cloog-0.18.0 /nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8 /nix/store/j81x6cd2zdqp1f0yprghgd266d5zgjx0-mpfr-3.1.2

    /nix/store/jqmfwcky6r6m177ak87rsrfg825z383w-zlib-1.2.8 /nix/store/m551ya45l9gg0wclllhniszrccl3szb6-mpc-1.0.1 /nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2 /nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3 /nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 store closure
  52. % nix-copy-closure --to anotherbox /nix/store/ lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 copying 4 missing paths

    to ‘anotherbox’... importing path `/nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8' load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s importing path `/nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2' importing path `/nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3' importing path `/nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82'
  53. More distributed builds (fanout through ssh) caching for free

  54. None
  55. NixOS binary channels user environments (profiles) /etc/nixos/configuration.nix generations / rollbacks

    Solaris-like "boot configurations" mostly designed for laptops / legacy systems :(
  56. configuration management

  57. misconfiguration management

  58. configuration mismanagement

  59. puppet chef / ansible / salt / whatever

  60. None
  61. None
  62. let region = "ap-southeast-1"; accessKeyId = "devops"; gen-instance = {

    resources, ... }: (import ./thumbor.nix {}) { deployment.ec2 = { inherit accessKeyId region; instanceType = "c3.2xlarge"; }; fileSystems."/data" = { ec2.disk = "ephemeral0"; }; }; in { resources.ec2KeyPairs.default = { inherit accessKeyId region; }; thumbor1 = gen-instance; thumbor2 = gen-instance; thumbor3 = gen-instance; thumbor4 = gen-instance; }
  63. ????

  64. None
  65. { lib }: with lib; rec { mapcat = f:

    x: concatLists (mapAttrsToList f x); mapcata = f: x: listToAttrs (mapcat f x); } maps/filters included
  66. resources.elbs.web = { resources, config, pkgs, lib, ... }: with

    lib; { inherit region accessKeyId; name = "${name}-web"; subnets = [ resources.subnets.default ]; securityGroups = [ resources.ec2SecurityGroups.default ]; machines = mapAttrsToList (name: m: m) resources.machines; route53Aliases = mapcata (_name: app: (map (prefix: { name = prefix + app.dns.public.name; value = { zoneId = app.dns.public.zoneId; }; }) ["" "*."])) all-apps; };
  67. Integration --eval --strict --xml builtins.toJSON builtins.fromJSON staged evaluations perl bindings

    (omg please don't)
  68. Future hnix https:/ /github.com/jwiegley/hnix

  69. More nix-docker cabal2nix python2nix hydra (Nix CI) guix smart community!

  70. Where it helps (aka show me the money)

  71. reconciling configuration hell (tooling helps)

  72. separating implementation and declaration while keeping code and configuration together

  73. managing dependency hell (e.g. sudden rewrites of services from Go

    to Python)
  74. bridging multilang dependencies (i.e. Python with C with Haskell)

  75. writing shell scripts!

  76. VPC IP<->Route53 mapping https://gist.github.com/proger/49e7ae550f4e2d4c99af

  77. zone=ZOZONEZONEZONE date=$(curl -s -I https://route53.amazonaws.com/date \ | awk '/^Date: /

    { sub("Date: ", "", $0); sub("\\r", "", $0); print $0 }') set -- $(curl -s http://169.254.169.254/latest/meta-data/ iam/security-credentials/dns \ | jq -r '.SecretAccessKey, .AccessKeyId, .Token') signature=$(echo -n $date \ | openssl dgst -binary -sha1 -hmac $1 \ | base64) auth_header="X-Amzn-Authorization: AWS3-HTTPS AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname=$(hostname).doge-networking-enterprises.com local_hostname=$(curl -s http://169.254.169.254/latest/ meta-data/local-hostname)
  78. zone date | awk sub("Date: ", "", $0); sub("\\r", "",

    $0); print $0 }' set iam/security-credentials/dns | jq -r signature | openssl dgst -binary -sha1 -hmac | base64 auth_header AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname local_hostname meta-data/local-hostname curl awk curl jq openssl coreutils curl bash
  79. bash = "${pkgs.bash}/bin/bash"; base64 = "${pkgs.coreutils}/bin/base64"; jq = "${pkgs.jq}/bin/jq"; curl

    = "${pkgs.curl}/bin/curl -s --retry --fail"; awk = "${pkgs.gawk}/bin/awk"; openssl = "${pkgs.openssl}/bin/openssl";
  80. zone date | awk sub("Date: ", "", $0); sub("\\r", "",

    $0); print $0 }' set iam/security-credentials/dns | jq -r signature | openssl dgst -binary -sha1 -hmac | base64 auth_header AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname local_hostname meta-data/local-hostname ${curl} ${awk} ${jq} ${openssl} ${base64} ${curl} ${curl} ${bash}
  81. speeding up Haskell development (benefits of cabal sandbox without its

    disadvantages)
  82. managing Cabal hell & GHC versions

  83. library isolation for C and C++ software (that's usually where

    containers shine, but works for us)
  84. teamwork & pull requests

  85. developer environments close to production

  86. lifting your legacy apps into the cloud (your apps don't

    need configuration, remember?)
  87. Getting started mix and match Nix with Docker Nix on

    Debian with Chef Nix on Debian with NixOps
  88. https:/ /github.com/zalora we're using and improving Nix

  89. stay tuned ask me anything @darkproger