Share
Embed
The Zalora Platform
The story behind the purely functional software delivery platform at Zalora.
Initially presented during DevOps Meetup #1 in Minsk.
Keywords: Nix, Haskell, PaaS.
Volodymyr Kyrylov
March 25, 2015
Transcript
75 slides
-
Teams > ~60 people on #devops > 85 on GitHub
> PHP, Go, Haskell, Python, Ruby, Nix, Java, Objective-C > PHP monolith going microservice-ish -
> Implement true DevOps > Ease PagerDuty burden > Decrease
MTTR > Managed environment to move fast and break things -
Tools > Compile-time verification > CI as a first-class citizen
> First-class identities & auth > Ship early > Proto implementation doesn't matter > Easily swappable -
Dev envs > Zero-config bootstrap > I want to hack
on a plane > I have no internet access > I hate waiting for stuff to compile > I should run whatever there is in production > I want to communicate my changes > It has to work -
> puppet language is not expressive enough > YAML? LOL
> ansible, salt, etc > languages with effects are too much -
% cat databass.yaml sg: db-alpha.infra.zalora.io my: db-psi.infra.zalora.io id: db-psi.infra.zalora.io hk:
db-psi.infra.zalora.io th: db-theta.infra.zalora.io ph: db-beta.infra.zalora.io vn: db-beta.infra.zalora.io -
> our model is country- centric > countries can share
servers > mariadb channels are server-centric -
> rewrite by hand? > use some other script to
translate models? > write a puppet plugin to do that? > find a better template engine > like, embed python into your tags > why not use an expressive language in the first place? -
db-slave-channels = let mapper = _: { db-name, masterhost, ...
}: nameValuePair (to-key master-host) { inherit master-host; databases = [ db-name ]; }; reducer = { name, value }: all: all // { ${name} = value // { databases = all.${name}.databases or [] ++ value.databases; }; }; in fold reducer {} (mapAttrsToList mapper conf); -
db-slave-channels = let mapper = _: { db-name, masterhost, ...
}: nameValuePair (to-key master-host) { inherit master-host; databases = [ db-name ]; }; reducer = { name, value }: all: all // { ${name} = value // { databases = all.${name}.databases or [] ++ value.databases; }; }; in fold reducer {} (mapAttrsToList mapper conf); BIG DATA-compatible -
Jenkins > Click through all the forms? > XML LOL
> Only operation-oriented API > String templates? > Can build a DSL! -
eris-sdk = erisJob { branch = "master"; shell = ''
export SLACK_CHANNEL='#eris-facepalm' export SLACK_TIMEOUT=5m bin/slack make sdk ''; ssh-keys = [ credentials.hydrabot ]; triggers = [ (ghprb-trigger "eris-sdk") (github-push-trigger) ]; }; -
mapAttrs' (realm-name: spec: deployJob "deploy-${realm-name}" { inherit realm-name; scm =
eris_master; permissions = with spec; { build = humans-can-build ++ others-can-build; }; ssh-keys = attrValues credentials; })) (filterAttrs is-deployable realms); -
choice-parameter = { name , description ? "" , choices
? [ "this is a list" ] }: (term "hudson.model.ChoiceParameterDefinition" null [ (term "name" null name) (term "description" null description) (term "choices" { class = "java.util.Arrays $ArrayList"; } [ (term "a" { class = "string-array"; } (map (term "string" null) choices)) ]) ]); -
Apps > how to build > how to run >
availability criteria > logging conventions for automatic aggregation and rotation > nginx vhost for webservices -
{ sdk, costa-conf, ... }: let inherit (import <eris/lib>) toLower
app; inherit (costa-conf) memcache-endpoint endpoint; costa-config = sdk.writeJSON "costa-${venture-fas}.conf" { Memcache = memcache-endpoint; Listen = endpoint; }; in app { command = "${sdk.costa}/bin/costa ${costa-config}"; nginxServer = '' # boilerplate omitted... location / { proxy_pass ${endpoint}; } ''; logging = auto; diagnostics.main = { http-ok = "${endpoint}/_c/healthcheck"; timeseries.enable = true; }; } -
http-ok = mkOption { type = types.nullOr types.str; description =
"HTTP URL that is supposed to return a 2xx or a 3xx response."; }; mysql-metric = mkOption { type = types.nullOr types.str; description = "MySQL query that returns one row with a single column which is a numeric value."; }; mysql-status = mkOption { type = types.nullOr types.str; description = "MySQL variable from SHOW STATUS that returns a numeric value."; }; -
memcached-stat = mkOption { type = types.nullOr (types.submodule ({ ...
}: { options = { key = mkOption { type = types.str; default = "total_connections"; }; target = mkOption { type = types.str; default = "localhost:11211"; }; }; })); description = "Memcached numeric statistic."; }; memcached-kvmetric = mkOption { type = types.nullOr (types.submodule ({ ... }: { options = { key = mkOption { type = types.str; default = "update_timestamp"; }; target = mkOption { type = types.str; default = "localhost:11211"; }; }; })); description = "Memcached key lookup that returns a numeric value."; }; -
if check.http-ok != null then '' curl --max-time ${timeout} -f
-sS -o /dev/null "${check.http- ok}" metric=$? '' else if check.mysql-metric != null then '' metric=$(${timeoutCmd} ${sdk.mariadb}/bin/mysql -h 127.0.0.1 - qrN -B < ${builtins.toFile "mysql-metric" check.mysql-metric}) '' else if check.mysql-status != null then '' metric=$(${timeoutCmd} ${sdk.mariadb}/bin/mysql -h 127.0.0.1 - qrN -B < ${builtins.toFile "mysql-status" '' select variable_value from information_schema.global_status where variable_name = '${check.mysql-status}' ''}) '' -
> Same semantics — different implementation > CLI for debugging
> CloudWatch for important timeseries > Datadog/InfluxDB/OpenTSDB... -
infra = { ec2-instance = { web1 = instance "m3.large";
web2 = instance "m3.large"; db-master = { infra, ... }: instance "r3.xlarge" // { blockDeviceMapping."/dev/xvdm".disk = infra.ebs.database; }; cron = instance "m3.large"; }; elb = elb.defaults { web = [ "web1" "web2" ]; }; ebs.database = { inherit (environment.ec2-args) region zone; size = 200; volumeType.gp2 = true; }; }; -
Specs > Evaluation expands specs by substituting variables > beta-reduced
term > You can print it and hand out to your cloud procurement manager -
Nix > pure > structured data goes in > data
goes out > no I/O > uni-typed > terms are isomorphic to JSON -
Nix > lazy > expression-oriented > HOFs, closures > data
structures > map/filter/fold/... > runtime type-checking support > via libraries -
Nix > single side effect > allows writing data to
CAS > allows running programs in CAS context (aka derivations) — > used to build a package manager > immutable > can't write a different thing under the same key -
error 16-Jan-2015 15:46:52 these derivations will be built: error 16-Jan-2015
15:46:52 /nix/store/qzsnk8plmdhfrx0vj426di3aqxy8qx22-seafas.drv error 16-Jan-2015 15:46:52 /nix/store/8mq0ciwyvnb6319xrs8xih7cafgifw5v-bob-cli-SGFAS.drv error 16-Jan-2015 15:46:52 /nix/store/3v5w6cizs7vfhg4gpp0ni9aadbyf8vzj-bob-schema-updater-SGFAS.drv error 16-Jan-2015 15:46:52 /nix/store/xc7q5azlfpzfaik40j3866b1k08sgjdd-bob-cli-MYFAS.drv error 16-Jan-2015 15:46:52 /nix/store/40ni45kmqirn67vgl7vj1dbmvwvbmd2z-bob-solr-import-MYFAS.drv error 16-Jan-2015 15:46:52 /nix/store/ygfg1j68bgd5mf9ifb3wiva4jmnarmb4-bob-cli-HKFAS.drv error 16-Jan-2015 15:46:52 /nix/store/4xhq1hvxsilvf3snkgr5k2qlzmsp67sd-bob-schema-updater-HKFAS.drv error 16-Jan-2015 15:46:52 /nix/store/f5jp2xr6walivg6ym89clk6vwnda5wzk-bob-cli-PHFAS.drv error 16-Jan-2015 15:46:52 /nix/store/66ppybs0x4nmfvn42iv4h05zkhpka0pl-bob-schema-updater-PHFAS.drv error 16-Jan-2015 15:46:52 /nix/store/793p7pyvlxhb2086ps98dgihcc3fvphg-bob-cli-VNFAS.drv error 16-Jan-2015 15:46:52 /nix/store/6rrz7v2a8sg9l8gf9nhj3sgc6w06qchv-bob-solr-import-VNFAS.drv error 16-Jan-2015 15:46:52 /nix/store/7a4cv80xw64kgjgx3jjgcgikvah5xvkp-bob-solr-update-HKFAS.drv error 16-Jan-2015 15:46:52 /nix/store/805ybbkyy2bpgc5rdxm98bzlpk1gr8wi-bob-schema-updater-VNFAS.drv error 16-Jan-2015 15:46:52 /nix/store/9jbs7mpflxcybqp4bfgq1c0lgcw7cs2z-bob-full-worker-SGFAS.drv error 16-Jan-2015 15:46:52 /nix/store/9snidxjs1r7nd4m4rl41rgma7ll42kl4-bob-full-worker-PHFAS.drv error 16-Jan-2015 15:46:52 /nix/store/ab67s87g21ssh26m2y0vwbpf83qb83iy-bob-schema-updater-MYFAS.drv error 16-Jan-2015 15:46:52 /nix/store/mayvq3rpcnp4nnc4vzv1jk11nj6728ii-bob-cli-THFAS.drv error 16-Jan-2015 15:46:52 /nix/store/ajnb1nxswz56h7yhf73iyhrp8z1d1jaz-bob-solr-update-THFAS.drv error 16-Jan-2015 15:46:52 /nix/store/ajwmssg6saabr83yycz4q2lqxrhgn9j2-bob-solr-import-HKFAS.drv error 16-Jan-2015 15:46:52 /nix/store/dgcvihqlr2anc8qg64i3v4aafwk0np6c-bob-full-worker-THFAS.drv error 16-Jan-2015 15:46:52 /nix/store/dslsry6wc22r7pj77dyhkpzxaphc5kjs-bob-full-worker-MYFAS.drv error 16-Jan-2015 15:46:52 /nix/store/dsz7jr3iwkdiff5ixj1aflfz5qmbgxq7-bob-solr-update-SGFAS.drv error 16-Jan-2015 15:46:52 /nix/store/dyqljj15vs4szvm9pv4k7wvygawgavmd-bob-solr-update-MYFAS.drv error 16-Jan-2015 15:46:52 /nix/store/l1amygiimqk1s7q63a5nplfaic5m2n97-bob-cli-IDFAS.drv error 16-Jan-2015 15:46:52 /nix/store/f0gli4fm148iknvqri1iwbyd6b4vs4dn-bob-solr-update-IDFAS.drv error 16-Jan-2015 15:46:52 /nix/store/klskg051hp9h1vp5lscrhhy372012qms-bob-solr-import-IDFAS.drv error 16-Jan-2015 15:46:52 /nix/store/l7nhn2i65qdgfa22vj268gdx1xljrwjn-bob-solr-import-THFAS.drv error 16-Jan-2015 15:46:52 /nix/store/md0agj0p9h9xqnrdzp81gddd221lkrd7-bob-solr-update-PHFAS.drv error 16-Jan-2015 15:46:52 /nix/store/mi3dbfw0wa4bm94lfngqsaqvivy6j0a1-bob-schema-updater-IDFAS.drv error 16-Jan-2015 15:46:52 /nix/store/mylyixmisbkczvsxdcham214wnngrsc8-bob-solr-import-PHFAS.drv error 16-Jan-2015 15:46:52 /nix/store/q1dfjmylhya2azjl19zs2ji7fq8lwkd7-bob-full-worker-VNFAS.drv error 16-Jan-2015 15:46:52 /nix/store/qqa2clnhglnpzzcnp490rcvc9llqbn58-bob-full-worker-IDFAS.drv error 16-Jan-2015 15:46:52 /nix/store/qy5hi49z99212k2g8nzbaqmy6rs5gc78-bob-full-worker-HKFAS.drv error 16-Jan-2015 15:46:52 /nix/store/s12hj5zslm2bnmyjvfb0g0mcg1b1hjl4-bob-solr-import-SGFAS.drv error 16-Jan-2015 15:46:52 /nix/store/wbfbdvchp1b6j4jj107lvlm6871lgx2l-bob-solr-update-VNFAS.drv error 16-Jan-2015 15:46:52 /nix/store/zkghcwd4viyyjwr3jm8vn16c23hp4cks-bob-schema-updater-THFAS.drv -
Deployments > interactive: > copy a store path closure to
target machine > good for SPOFs > immutable: > ASGs, etc -
Image backends > EC2 AMI > VirtualBox VDI/OVA > VMWare
> Vagrant > Docker images > ... > implementable using Nix derivations -
Upcast > compiles expanded Nix specs into AWS API calls
> handles interactive/partial deployments -
Platform > data (configs) > Linux defaults > the SDK
(software collection) > infra evaluation (upcast) > integration evaluation (jenkins, pagerduty, etc) > autoscaling-ready > internal part is called "Eris" -
> strict separation of control plane vs data plane >
semantics first, implementation second > multi-cloud > multi-OS > etc -
> taking irrelevant details out of the equation > moving
them to another teams or subprojects -
> simplify doing business in the real world without going
into tech details too much > perfect for wrapping stringly-typed Linux world and unifying interfaces -
> adding effectful computations into the language (IO monad) >
better take a proper language instead > haven't found one yet