Функциональный тулчейн Nix

Transcript

1. λ-toolchain

2. λ-toolchain @darkproger (initially presented on Most Functional Day; Aug 9,

   2014)

3. Nix purely functional package & configuration mgr programming language

4. Nix apt-get / yum json + macros puppet / chef

   {

5. Nix apt-get / yum json + macros puppet / chef

6. the language

7. None
8. None

9. Nix (the expression language) lazy expression-oriented HOFs, closures

10. None

11. % nix-instantiate --eval -E 1 1

12. % nix-instantiate --eval -E ' builtins.trace "hello world" null '

    trace: hello world null

13. % nix-instantiate --eval -E ' [ 1 2 3 ]

    ' [ 1 2 3 ] lists

14. % nix-instantiate --eval -E ' { hello = "world"; one

    = 1; } ' { hello = "world"; one = 1; } attribute sets

15. % nix-instantiate --eval -E ' rec { hello = "world";

    again = hello; } ' { again = <CODE>; hello = "world"; } recursive attrsets

16. % nix-instantiate --eval --strict -E ' rec { hello =

    "world"; again = hello; } ' { again = "world"; hello = "world"; } (optional) strict evaluation

17. % nix-instantiate --eval -E ' (a: "hello ${a}") "john doe"

    ' "hello john doe" lambdas

18. % nix-instantiate --eval -E ' ((a: b: "hello ${a} ${b}")

    "john") ' <LAMBDA> currying

19. % nix-instantiate --eval -E ' (((a: b: "hello ${a} ${b}")

    "john") "doe") ' "hello john doe" currying

20. % nix-repl Welcome to Nix version 1.7. Type :? for

    help. nix-repl> :p let a = "one"; b = "two"; f = { foo, bar }: "${foo}: ${bar}”; in f { foo = a; bar = b; } "one: two" argument deconstruction

21. nix-repl> :p let foo = “one”; bar = “two"; f

    = { foo, bar }: "${foo}: ${bar}”; in f { inherit foo bar; } "one: two" scope inheritance

22. expression files

23. { lib, ... }: with lib; let ec2 = {

    "my-elastic-ip" = "54.127.128.129/32"; }; rackspace = { "huge-box" = "154.24.23.52/32"; "build-box" = "165.123.32.23/32"; }; locations = { "home" = "127.0.0.1/32"; "office" = "169.254.169.254/32"; }; in rec { netmap = ec2 // rackspace // locations; whitelist = attrValues netmap; }

24. { lib, ... }: with lib; let ec2 = {

    "my-elastic-ip" = "54.127.128.129/32"; }; rackspace = { "huge-box" = "154.24.23.52/32"; "build-box" = "165.123.32.23/32"; }; locations = { "home" = "127.0.0.1/32"; "office" = "169.254.169.254/32"; }; in rec { netmap = ec2 // rackspace // locations; whitelist = attrValues netmap; }

25. % nix-instantiate --eval my-networks.nix <LAMBDA>

26. % nix-instantiate --eval my-networks.nix <LAMBDA> { lib, ... }: with

    lib;

27. % nix-instantiate --eval \ --arg lib '(import <nixpkgs> {}).lib' \

    my-networks.nix { netmap = <CODE>; whitelist = <CODE>; }

28. % nix-instantiate --eval --strict \ --arg lib '(import <nixpkgs> {}).lib'

    \ my-networks.nix { netmap = { build-box = "165.123.32.23/32"; home = "127.0.0.1/32"; huge-box = "154.24.23.52/32"; my-elastic-ip = "54.127.128.129/32"; office = "169.254.169.254/32"; }; whitelist = [ "165.123.32.23/32" "127.0.0.1/32" "154.24.23.52/32" "54.127.128.129/32" "169.254.169.254/32" ]; }

29. % nix-instantiate --eval --strict -E ' builtins.toJSON import ./my-networks.nix {

    inherit (import <nixpkgs> {}) lib; } ' error: cannot convert a partially applied built- in function to JSON

30. % nix-instantiate --eval --strict -E ' builtins.toJSON (import ./my-networks.nix {

    inherit (import <nixpkgs> {}) lib; }) ' | jq -r . | jq . { "whitelist": [ "165.123.32.23/32", "127.0.0.1/32", "154.24.23.52/32", "54.127.128.129/32", "169.254.169.254/32" ], "netmap": { "office": "169.254.169.254/32", "my-elastic-ip": "54.127.128.129/32", "huge-box": "154.24.23.52/32", "home": "127.0.0.1/32", "build-box": "165.123.32.23/32" } }

31. '' #!${pkgs.bash}/bin/bash # this string is going to be indented

    cd ${if cfg.dataDir != "" then cfg.dataDir else "/var/empty"} exec ${cfg.command} ''; smart interpolation

32. '' #!${pkgs.bash}/bin/bash # this string is going to be indented

    cd ${if cfg.dataDir != "" then cfg.dataDir else "/var/empty"} exec ${cfg.command} ''; "antiquotation"

33. derivations

34. let system = builtins.currentSystem; builder = /bin/bash; in rec {

    writeText = name: text: derivation { inherit system builder name; args = ["-c" "echo -n '${text}' > $out"]; }; }

35. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning:

    you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv

36. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning:

    you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv the store

37. % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning:

    you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv % nix-instantiate -E ' (import ./ourlib.nix).writeText "hello" "world" ' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv

38. Derive( [("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello","","")], [], ["/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"], "x86_64-darwin", "/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash", ["-c","echo -n 'world' >

    $out"], [("builder","/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash"), ("name","hello"), ("out","/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello"), ("system","x86_64-darwin")] ) "annotated term"

39. % nix-store --realise /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv these derivations will be built:

    /nix/store/ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv building path(s) `/nix/store/ p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello' warning: you did not specify `--add-root'; the result might be removed by the garbage collector /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello realising the derivation

40. % nix-store --realise /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv warning: you did not specify

    `--add-root'; the result might be removed by the garbage collector /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello ... only once

41. % nix-store --print-env /nix/store/ ic243k5dw29knvgp60x8nh22kmkbg4dh-hello.drv export builder; builder='/nix/store/mv24aqvq44jvdxnhszx6kyps9jlf0ial-bash' export name;

    name='hello' export out; out='/nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello' export system; system='x86_64-darwin' export _args; _args='-c echo -n '\''world'\'' > $out' reproducible env

42. % nix-shell --pure -E ' (import ./ourlib.nix).writeText "hello" "world" '

    [nix-shell:~]$ nix-shell

43. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello world

44. % cat /nix/store/p30mrskrnk8y5zvknc7ccppz3m30vkbf-hello world hash of all inputs

45. derivations package manager

46. None

47. % ll $(nix-instantiate --eval -E '<nixpkgs>')/ total 24 -rw-r--r-- 1

    proger staff 1.6K Jun 17 13:36 COPYING -rw-r--r-- 1 proger staff 677B Jun 23 17:17 README.md -rw-r--r-- 1 proger staff 213B Jun 17 13:36 default.nix drwxr-xr-x 15 proger staff 510B Jul 21 10:17 doc/ drwxr-xr-x 22 proger staff 748B Jul 21 10:17 lib/ drwxr-xr-x 3 proger staff 102B Jun 17 13:36 maintainers/ drwxr-xr-x 13 proger staff 442B Jul 21 10:17 nixos/ drwxr-xr-x 16 proger staff 544B Jun 17 13:36 pkgs/

48. % cat ~dev/nix/nixpkgs/pkgs/os-specific/linux/latencytop/default.nix { stdenv, fetchurl, ncurses, glib, pkgconfig, gtk

    }: stdenv.mkDerivation rec { name = "latencytop-0.5"; patchPhase = "sed -i s,/usr,$out, Makefile"; preInstall = "mkdir -p $out/sbin"; src = fetchurl { urls = [ "http://latencytop.org/download/${name}.tar.gz" "http://dbg.download.sourcemage.org/mirror/latencytop-0.5.tar.gz" ]; sha256 = "1vq3j9zdab6njly2wp900b3d5244mnxfm88j2bkiinbvxbxp4zwy"; }; buildInputs = [ ncurses glib pkgconfig gtk ]; meta = { homepage = http://latencytop.org; description = "Tool to show kernel reports on latencies (LATENCYTOP option)"; license = "GPLv2"; maintainers = [ stdenv.lib.maintainers.viric ]; platforms = stdenv.lib.platforms.linux; }; }

49. stdenv basic toolchain for C ./configure --prefix=$out make make install

    (and more) unix idiosyncrasies unix flavours monkey-patching (patchelf) nix-shell matters even more

50. stdenv for $LANGUAGE build on top of language- specific package

    manager integrate with the rest of the world

51. % nix-store -qR /nix/store/ lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 /nix/store/x52mfsvjaz1k82iy8hh5bmh1jhz62kwg-gmp-5.1.3 /nix/store/q4dp57r561pqps48sqnps74xwz9fg9b4-isl-0.11.1 /nix/store/939lir5rs9axn2pch7sxgxwpvk2yq20d-cloog-0.18.0 /nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8 /nix/store/j81x6cd2zdqp1f0yprghgd266d5zgjx0-mpfr-3.1.2

    /nix/store/jqmfwcky6r6m177ak87rsrfg825z383w-zlib-1.2.8 /nix/store/m551ya45l9gg0wclllhniszrccl3szb6-mpc-1.0.1 /nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2 /nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3 /nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 store closure

52. % nix-copy-closure --to anotherbox /nix/store/ lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82 copying 4 missing paths

    to ‘anotherbox’... importing path `/nix/store/fz99gjn0vi6nmxnv2rr6kh2sfc85mpy2-zlib-1.2.8' load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s load: 1.23 cmd: ssh 24006 waiting 0.00u 0.00s importing path `/nix/store/yy6px6c2fxmq4bcs6mzq16wfnlc9l36y-gcc-4.8.2' importing path `/nix/store/kif3h9a68a6vc3r7s7y0i3cszjjix21q-luajit-2.0.3' importing path `/nix/store/lfmx59j4ds9k7f6grs6l5y7y7wq52d3n-sysdig-0.1.82'

53. More distributed builds (fanout through ssh) caching for free

54. None

55. NixOS binary channels user environments (profiles) /etc/nixos/configuration.nix generations / rollbacks

    Solaris-like "boot configurations" mostly designed for laptops / legacy systems :(

56. configuration management

57. misconfiguration management

58. configuration mismanagement

59. puppet chef / ansible / salt / whatever

60. None
61. None

62. let region = "ap-southeast-1"; accessKeyId = "devops"; gen-instance = {

    resources, ... }: (import ./thumbor.nix {}) { deployment.ec2 = { inherit accessKeyId region; instanceType = "c3.2xlarge"; }; fileSystems."/data" = { ec2.disk = "ephemeral0"; }; }; in { resources.ec2KeyPairs.default = { inherit accessKeyId region; }; thumbor1 = gen-instance; thumbor2 = gen-instance; thumbor3 = gen-instance; thumbor4 = gen-instance; }

63. ????

64. None

65. { lib }: with lib; rec { mapcat = f:

    x: concatLists (mapAttrsToList f x); mapcata = f: x: listToAttrs (mapcat f x); } maps/filters included

66. resources.elbs.web = { resources, config, pkgs, lib, ... }: with

    lib; { inherit region accessKeyId; name = "${name}-web"; subnets = [ resources.subnets.default ]; securityGroups = [ resources.ec2SecurityGroups.default ]; machines = mapAttrsToList (name: m: m) resources.machines; route53Aliases = mapcata (_name: app: (map (prefix: { name = prefix + app.dns.public.name; value = { zoneId = app.dns.public.zoneId; }; }) ["" "*."])) all-apps; };

67. Integration --eval --strict --xml builtins.toJSON builtins.fromJSON staged evaluations perl bindings

    (omg please don't)

68. Future hnix https:/ /github.com/jwiegley/hnix

69. More nix-docker cabal2nix python2nix hydra (Nix CI) guix smart community!

70. Where it helps (aka show me the money)

71. reconciling configuration hell (tooling helps)

72. separating implementation and declaration while keeping code and configuration together

73. managing dependency hell (e.g. sudden rewrites of services from Go

    to Python)

74. bridging multilang dependencies (i.e. Python with C with Haskell)

75. writing shell scripts!

76. VPC IP<->Route53 mapping https://gist.github.com/proger/49e7ae550f4e2d4c99af

77. zone=ZOZONEZONEZONE date=$(curl -s -I https://route53.amazonaws.com/date \ | awk '/^Date: /

    { sub("Date: ", "", $0); sub("\\r", "", $0); print $0 }') set -- $(curl -s http://169.254.169.254/latest/meta-data/ iam/security-credentials/dns \ | jq -r '.SecretAccessKey, .AccessKeyId, .Token') signature=$(echo -n $date \ | openssl dgst -binary -sha1 -hmac $1 \ | base64) auth_header="X-Amzn-Authorization: AWS3-HTTPS AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname=$(hostname).doge-networking-enterprises.com local_hostname=$(curl -s http://169.254.169.254/latest/ meta-data/local-hostname)

78. zone date | awk sub("Date: ", "", $0); sub("\\r", "",

    $0); print $0 }' set iam/security-credentials/dns | jq -r signature | openssl dgst -binary -sha1 -hmac | base64 auth_header AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname local_hostname meta-data/local-hostname curl awk curl jq openssl coreutils curl bash

79. bash = "${pkgs.bash}/bin/bash"; base64 = "${pkgs.coreutils}/bin/base64"; jq = "${pkgs.jq}/bin/jq"; curl

    = "${pkgs.curl}/bin/curl -s --retry --fail"; awk = "${pkgs.gawk}/bin/awk"; openssl = "${pkgs.openssl}/bin/openssl";

80. zone date | awk sub("Date: ", "", $0); sub("\\r", "",

    $0); print $0 }' set iam/security-credentials/dns | jq -r signature | openssl dgst -binary -sha1 -hmac | base64 auth_header AWSAccessKeyId=$2,Algorithm=HmacSHA1,Signature=$signature" hostname local_hostname meta-data/local-hostname ${curl} ${awk} ${jq} ${openssl} ${base64} ${curl} ${curl} ${bash}

81. speeding up Haskell development (benefits of cabal sandbox without its

    disadvantages)

82. managing Cabal hell & GHC versions

83. library isolation for C and C++ software (that's usually where

    containers shine, but works for us)

84. teamwork & pull requests

85. developer environments close to production

86. lifting your legacy apps into the cloud (your apps don't

    need configuration, remember?)

87. Getting started mix and match Nix with Docker Nix on

    Debian with Chef Nix on Debian with NixOps

88. https:/ /github.com/zalora we're using and improving Nix

89. stay tuned ask me anything @darkproger