Nix: λ-toolchain

Transcript

1. λ-toolchain
2. λ-toolchain @darkproger (initially presented on Most Functional Day; Aug 9, 2014)
3. Nix purely functional package & configuration mgr programming language
4. Nix apt-get / yum json + macros puppet / chef {
5. Nix apt-get / yum json + macros puppet / chef
6. the language
9. Nix (the expression language) lazy expression-oriented HOFs, closures
11. % nix-instantiate --eval -E 1 1
12. % nix-instantiate --eval -E ' builtins.trace "hello world" null ' trace: hello world null
13. % nix-instantiate --eval -E ' [ 1 2 3 ] ' [ 1 2 3 ] lists
14. % nix-instantiate --eval -E ' { hello = "world"; one = 1; } ' { hello = "world"; one = 1; } attribute sets
15. % nix-instantiate --eval -E ' rec { hello = "world"; again = hello; } ' { again = <CODE>; hello = "world"; } recursive attrsets
16. % nix-instantiate --eval --strict -E ' rec { hello = "world"; again = hello; } ' { again = "world"; hello = "world"; } (optional) strict evaluation
17. % nix-instantiate --eval -E ' (a: "hello ${a}") "john doe" ' "hello john doe" lambdas
18. % nix-instantiate --eval -E ' ((a: b: "hello ${a} ${b}") "john") ' <LAMBDA> currying
19. % nix-instantiate --eval -E ' (((a: b: "hello ${a} ${b}") "john") "doe") ' "hello john doe" currying
20. % nix-repl Welcome to Nix version 1.7. Type :? for help. nix-repl> :p let a = "one"; b = "two"; f = { foo, bar }: "${foo}: ${bar}”; in f { foo = a; bar = b; } "one: two" argument deconstruction
21. nix-repl> :p let foo = “one”; bar = “two"; f = { foo, bar }: "${foo}: ${bar}”; in f { inherit foo bar; } "one: two" scope inheritance
22. expression files
23. { lib, ... }: with lib; let ec2 = { "my-elastic-ip" = "54.127.128.129/32"; }; rackspace = { "huge-box" = "154.24.23.52/32"; "build-box" = "165.123.32.23/32"; }; locations = { "home" = "127.0.0.1/32"; "office" = "169.254.169.254/32"; }; in rec { netmap = ec2 // rackspace // locations; whitelist = attrValues netmap; }
24. { lib, ... }: with lib; let ec2 = { "my-elastic-ip" = "54.127.128.129/32"; }; rackspace = { "huge-box" = "154.24.23.52/32"; "build-box" = "165.123.32.23/32"; }; locations = { "home" = "127.0.0.1/32"; "office" = "169.254.169.254/32"; }; in rec { netmap = ec2 // rackspace // locations; whitelist = attrValues netmap; }
25. % nix-instantiate --eval my-networks.nix <LAMBDA>
26. % nix-instantiate --eval my-networks.nix <LAMBDA> { lib, ... }: with lib;
27. % nix-instantiate --eval \ --arg lib '(import <nixpkgs> {}).lib' \ my-networks.nix { netmap = <CODE>; whitelist = <CODE>; }
28. % nix-instantiate --eval --strict \ --arg lib '(import <nixpkgs> {}).lib' \ my-networks.nix { netmap = { build-box = "165.123.32.23/32"; home = "127.0.0.1/32"; huge-box = "154.24.23.52/32"; my-elastic-ip = "54.127.128.129/32"; office = "169.254.169.254/32"; }; whitelist = [ "165.123.32.23/32" "127.0.0.1/32" "154.24.23.52/32" "54.127.128.129/32" "169.254.169.254/32" ]; }
29. % nix-instantiate --eval --strict -E ' builtins.toJSON import ./my-networks.nix { inherit (import <nixpkgs> {}) lib; } ' error: cannot convert a partially applied built- in function to JSON
30. % nix-instantiate --eval --strict -E ' builtins.toJSON (import ./my-networks.nix { inherit (import <nixpkgs> {}) lib; }) ' | jq -r . | jq . { "whitelist": [ "165.123.32.23/32", "127.0.0.1/32", "154.24.23.52/32", "54.127.128.129/32", "169.254.169.254/32" ], "netmap": { "office": "169.254.169.254/32", "my-elastic-ip": "54.127.128.129/32", "huge-box": "154.24.23.52/32", "home": "127.0.0.1/32", "build-box": "165.123.32.23/32" } }
31. '' #!${pkgs.bash}/bin/bash # this string is going to be indented cd ${if cfg.dataDir != "" then cfg.dataDir else "/var/empty"} exec ${cfg.command} ''; smart interpolation
32. '' #!${pkgs.bash}/bin/bash # this string is going to be indented cd ${if cfg.dataDir != "" then cfg.dataDir else "/var/empty"} exec ${cfg.command} ''; "antiquotation"
33. derivations
34. let system = builtins.currentSystem; builder = /bin/bash; in rec { writeText = name: text: derivation { inherit system builder name; args = ["-c" "echo -n '${text}' > $out"]; }; }
35. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
36. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv the store
37. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv
38. Derive( [("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello","","")], [], ["/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"], "x86_64-darwin", "/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash", ["-c","echo -n 'world' > $out"], [("builder","/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"), ("name","hello"), ("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello"), ("system","x86_64-darwin")] ) "annotated term"
39. % nix-store --realise /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv these derivations will be built: /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv building path(s) `/nix/store/ p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello realising the derivation
40. % nix-store --realise /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello ... only once
41. % nix-store --print-env /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv export builder; builder='/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash' export name; name='hello' export out; out='/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello' export system; system='x86_64-darwin' export _args; _args='-c echo -n '\''world'\'' > $out' reproducible env
42. % nix-shell --pure -E ' (import ./ourlib.nix).writeText "hello" "world" ' [nix-shell:~]$ nix-shell
43. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello world
44. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello world hash of all inputs
45. derivations package manager
47. % ll $(nix-instantiate --eval -E '<nixpkgs>')/ total 24 -rw-r--r-- 1 proger staff 1.6K Jun 17 13:36 COPYING -rw-r--r-- 1 proger staff 677B Jun 23 17:17 README.md -rw-r--r-- 1 proger staff 213B Jun 17 13:36 default.nix drwxr-xr-x 15 proger staff 510B Jul 21 10:17 doc/ drwxr-xr-x 22 proger staff 748B Jul 21 10:17 lib/ drwxr-xr-x 3 proger staff 102B Jun 17 13:36 maintainers/ drwxr-xr-x 13 proger staff 442B Jul 21 10:17 nixos/ drwxr-xr-x 16 proger staff 544B Jun 17 13:36 pkgs/
48. % cat ~dev/nix/nixpkgs/pkgs/os-specific/linux/latencytop/default.nix { stdenv, fetchurl, ncurses, glib, pkgconfig, gtk }: stdenv.mkDerivation rec { name = "latencytop-0.5"; patchPhase = "sed -i s,/usr,$out, Makefile"; preInstall = "mkdir -p $out/sbin"; src = fetchurl { urls = [ "http://latencytop.org/download/${name}.tar.gz" "http://dbg.download.sourcemage.org/mirror/latencytop-0.5.tar.gz" ]; sha256 = "1vq3j9zdab6njly2wp900b3d5244mnxfm88j2bkiinbvxbxp4zwy"; }; buildInputs = [ ncurses glib pkgconfig gtk ]; meta = { homepage = http://latencytop.org; description = "Tool to show kernel reports on latencies (LATENCYTOP option)"; license = "GPLv2"; maintainers = [ stdenv.lib.maintainers.viric ]; platforms = stdenv.lib.platforms.linux; }; }
49. stdenv basic toolchain for C ./configure --prefix=$out make make install (and more) unix idiosyncrasies unix flavours monkey-patching (patchelf) nix-shell matters even more
50. stdenv for $LANGUAGE build on top of language- specific package manager integrate with the rest of the world
51. % nix-store -qR /nix/store/ lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 /nix/store/x52mfsvjaz1k82iy8hh5bmh1jhz62kwg-gmp-5.1.3 /nix/store/q4dp57r561pqps48sqnps74xwz9fg9b4-isl-0.11.1 /nix/store/939lir5rs9axn2pch7sxgxwpvk2yq20d-cloog-0.18.0 /nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8 /nix/store/j81x6cd2zdqp1f0yprghgd266d5zgjx0-mpfr-3.1.2 /nix/store/jqmfwcky6r6m177ak87rsrfg825z383w-zlib-1.2.8 /nix/store/m551ya45l9gg0wclllhniszrccl3szb6-mpc-1.0.1 /nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2 /nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3 /nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 store closure
52. % nix-copy-closure --to anotherbox /nix/store/ lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 copying 4 missing paths to ‘anotherbox’... importing path `/nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8' load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s importing path `/nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2' importing path `/nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3' importing path `/nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82'
53. More distributed builds (fanout through ssh) caching for free
55. NixOS binary channels user environments (profiles) /etc/nixos/configuration.nix generations / rollbacks Solaris-like "boot configurations" mostly designed for laptops / legacy systems :(
56. configuration management
57. misconfiguration management
58. configuration mismanagement
59. puppet chef / ansible / salt / whatever
62. let region = "ap-southeast-1"; accessKeyId = "devops"; gen-instance = { resources, ... }: (import ./thumbor.nix {}) { deployment.ec2 = { inherit accessKeyId region; instanceType = "c3.2xlarge"; }; fileSystems."/data" = { ec2.disk = "ephemeral0"; }; }; in { resources.ec2KeyPairs.default = { inherit accessKeyId region; }; thumbor1 = gen-instance; thumbor2 = gen-instance; thumbor3 = gen-instance; thumbor4 = gen-instance; }
63. ????
65. { lib }: with lib; rec { mapcat = f: x: concatLists (mapAttrsToList f x); mapcata = f: x: listToAttrs (mapcat f x); } maps/filters included
66. resources.elbs.web = { resources, config, pkgs, lib, ... }: with lib; { inherit region accessKeyId; name = "${name}-web"; subnets = [ resources.subnets.default ]; securityGroups = [ resources.ec2SecurityGroups.default ]; machines = mapAttrsToList (name: m: m) resources.machines; route53Aliases = mapcata (_name: app: (map (prefix: { name = prefix + app.dns.public.name; value = { zoneId = app.dns.public.zoneId; }; }) ["" "*."])) all-apps; };
67. Integration --eval --strict --xml builtins.toJSON builtins.fromJSON staged evaluations perl bindings (omg please don't)
68. Future hnix https:/ /github.com/jwiegley/hnix
69. More nix-docker cabal2nix python2nix hydra (Nix CI) guix smart community!
70. Where it helps (aka show me the money)
71. reconciling configuration hell (tooling helps)
72. separating implementation and declaration while keeping code and configuration together
73. managing dependency hell (e.g. sudden rewrites of services from Go to Python)
74. bridging multilang dependencies (i.e. Python with C with Haskell)
75. writing shell scripts!
76. VPC IP<->Route53 mapping https://gist.github.com/proger/49e7ae550f4e2d4c99af
77. zone=ZOZONEZONEZONE date=$(curl -s -I https://route53.amazonaws.com/date \ | awk '/^Date: / { sub("Date: ", "", $0); sub("\\r", "", $0); print $0 }') set -- $(curl -s http://169.254.169.254/latest/meta-data/ iam/security-credentials/dns \ | jq -r '.SecretAccessKey, .AccessKeyId, .Token') signature=$(echo -n $date \ | openssl dgst -binary -sha1 -hmac $1 \ | base64) auth_header="X-Amzn-Authorization: AWS3-HTTPS AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname=$(hostname).doge-networking-enterprises.com local_hostname=$(curl -s http://169.254.169.254/latest/ meta-data/local-hostname)
78. zone date | awk sub("Date: ", "", $0); sub("\\r", "", $0); print $0 }' set iam/security-credentials/dns | jq -r signature | openssl dgst -binary -sha1 -hmac | base64 auth_header AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname local_hostname meta-data/local-hostname curl awk curl jq openssl coreutils curl bash
79. bash = "${pkgs.bash}/bin/bash"; base64 = "${pkgs.coreutils}/bin/base64"; jq = "${pkgs.jq}/bin/jq"; curl = "${pkgs.curl}/bin/curl -s --retry --fail"; awk = "${pkgs.gawk}/bin/awk"; openssl = "${pkgs.openssl}/bin/openssl";
80. zone date | awk sub("Date: ", "", $0); sub("\\r", "", $0); print $0 }' set iam/security-credentials/dns | jq -r signature | openssl dgst -binary -sha1 -hmac | base64 auth_header AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname local_hostname meta-data/local-hostname ${curl} ${awk} ${jq} ${openssl} ${base64} ${curl} ${curl} ${bash}
81. speeding up Haskell development (benefits of cabal sandbox without its disadvantages)
82. managing Cabal hell & GHC versions
83. library isolation for C and C++ software (that's usually where containers shine, but works for us)
84. teamwork & pull requests
85. developer environments close to production
86. lifting your legacy apps into the cloud (your apps don't need configuration, remember?)
87. Getting started mix and match Nix with Docker Nix on Debian with Chef Nix on Debian with NixOps
88. https:/ /github.com/zalora we're using and improving Nix
89. stay tuned ask me anything @darkproger