their environment and coordinate their actions thru a globally shared state” Lukasz Guminski, Container Solutions http://container-solutions.com/containerpilot-on-mantl/
or physical hardware Nginx Consul MySQL Primary ES Master Prometheus Logstash Customers Nginx Consul MySQL Primary ES Master Prometheus Logstash Customers Customers Cluster management & provisioning
Consul, # but do not reload because Nginx has't started yet preStart() { consul-template \ -once \ -consul consul:8500 \ -template "/etc/containerpilot/nginx.conf.ctmpl:/etc/nginx/nginx.conf" } # Render Nginx configuration template using values from Consul, # then gracefully reload Nginx onChange() { consul-template \ -once \ -consul consul:8500 \ -template "/etc/containerpilot/nginx.conf.ctmpl:/etc/nginx/nginx.conf:nginx -s reload" } until cmd=$1 if [ -z "$cmd" ]; then onChange fi shift 1 $cmd "$@" [ "$?" -ne 127 ] do onChange exit done ~/workshop/nginx/reload-nginx.sh /etc/containerpilot/nginx.conf.ctmpl:/etc/nginx/nginx.conf:nginx -s reload render this file : to this file : and then do this
write the address:port pairs for each healthy Sales node {{range service "sales"}} server {{.Address}}:{{.Port}}; {{end}} least_conn; }{{ end }} server { listen 80; server_name _; root /usr/share/nginx/html; {{ if service "sales" }} location ^~ /sales { # strip '/sales' from the request before passing # it along to the Sales upstream rewrite ^/sales(/.*)$ $1 break; proxy_pass http://sales; proxy_redirect off; }{{end}} } }
write the address:port pairs for each healthy Sales node {{range service "sales"}} server {{.Address}}:{{.Port}}; {{end}} least_conn; }{{ end }} server { listen 80; server_name _; root /usr/share/nginx/html; {{ if service "sales" }} location ^~ /sales { # strip '/sales' from the request before passing # it along to the Sales upstream rewrite ^/sales(/.*)$ $1 break; proxy_pass http://sales; proxy_redirect off; }{{end}} } }
write the address:port pairs for each healthy Sales node {{range service "sales"}} server {{.Address}}:{{.Port}}; {{end}} least_conn; }{{ end }} server { listen 80; server_name _; root /usr/share/nginx/html; {{ if service "sales" }} location ^~ /sales { # strip '/sales' from the request before passing # it along to the Sales upstream rewrite ^/sales(/.*)$ $1 break; proxy_pass http://sales; proxy_redirect off; }{{end}} } }
each healthy Sales node server 192.168.1.101:3000; server 192.168.1.102:3000; server 192.168.1.103:3000; least_conn; } server { listen 80; server_name _; root /usr/share/nginx/html; location ^~ /sales { # strip '/sales' from the request before passing # it along to the Sales upstream rewrite ^/sales(/.*)$ $1 break; proxy_pass http://sales; proxy_redirect off; } } }
{ // get data from Consul // fill upstreamHosts // fire callback } process.on('SIGHUP', function () { console.log('Received SIGHUP'); getUpstreams(true, function(hosts) { console.log(‘Updated upstreamHosts'); }); }); ~/workshop/sales/sales.js
expose for linking, but each container gets a private IP for # internal use as well expose: - 3306 labels: - triton.cns.services=mysql env_file: _env environment: - CONTAINERPILOT=file:///etc/containerpilot.json
expose for linking, but each container gets a private IP for # internal use as well expose: - 3306 labels: - triton.cns.services=mysql env_file: _env environment: - CONTAINERPILOT=file:///etc/containerpilot.json Infrastructure-backed service discovery requirement
expose for linking, but each container gets a private IP for # internal use as well expose: - 3306 labels: - triton.cns.services=mysql env_file: _env environment: - CONTAINERPILOT=file:///etc/containerpilot.json Credentials from environment
~/mysql $ docker-compose -p my ps Name Command State Ports ––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––– my_consul_1 /bin/start -server -bootst... Up 53/tcp, 53/udp, 8300/tcp... my_mysql_1 containerpilot mysqld… Up 0.0.0.0:3600 my_mysql_2 containerpilot mysqld… Up 0.0.0.0:3600
order to execute most of our setup behavior so we're just going to make sure the directory structures are in place and then let the first health check handler take it from there """ if not os.path.isdir(os.path.join(config.datadir, 'mysql')): last_backup = has_snapshot() if last_backup: get_snapshot(last_backup) restore_from_snapshot(last_backup) else: if not initialize_db(): log.info('Skipping database setup.') sys.exit(0)
order to execute most of our setup behavior so we're just going to make sure the directory structures are in place and then let the first health check handler take it from there """ if not os.path.isdir(os.path.join(config.datadir, 'mysql')): last_backup = has_snapshot() if last_backup: get_snapshot(last_backup) restore_from_snapshot(last_backup) else: if not initialize_db(): log.info('Skipping database setup.') sys.exit(0) Check w/ Consul for snapshot
order to execute most of our setup behavior so we're just going to make sure the directory structures are in place and then let the first health check handler take it from there """ if not os.path.isdir(os.path.join(config.datadir, 'mysql')): last_backup = has_snapshot() if last_backup: get_snapshot(last_backup) restore_from_snapshot(last_backup) else: if not initialize_db(): log.info('Skipping database setup.') sys.exit(0) calls /usr/bin/mysql_install_db
order to execute most of our setup behavior so we're just going to make sure the directory structures are in place and then let the first health check handler take it from there """ if not os.path.isdir(os.path.join(config.datadir, 'mysql')): last_backup = has_snapshot() if last_backup: get_snapshot(last_backup) restore_from_snapshot(last_backup) else: if not initialize_db(): log.info('Skipping database setup.') sys.exit(0)
Also acts as a check for whether the ContainerPilot configuration needs to be reloaded (if it's been changed externally), or if we need to make a backup because the backup TTL has expired. """ node = MySQLNode() cp = ContainerPilot(node) if cp.update(): cp.reload() return # Because we need MySQL up to finish initialization, we need to check # for each pass thru the health check that we've done so. The happy # path is to check a lock file against the node state (which has been # set above) and immediately return when we discover the lock exists. # Otherwise, we bootstrap the instance. was_ready = assert_initialized_for_state(node) ctx = dict(user=config.repl_user, password=config.repl_password, timeout=cp.config['services'][0]['ttl']) node.conn = wait_for_connection(**ctx) # Update our lock on being the primary/standby. if node.is_primary() or node.is_standby(): update_session_ttl() # Create a snapshot and send it to the object store if all((node.is_snapshot_node(), (not is_backup_running()), (is_binlog_stale(node.conn) or is_time_for_snapshot()))): write_snapshot(node.conn) mysql_query(node.conn, 'SELECT 1', ())
ported and reworked from the Oracle-provided Docker image: https://github.com/mysql/mysql-docker/blob/mysql-server/5.7/docker-entrypoint.sh """ node.state = PRIMARY mark_as_primary(node) node.conn = wait_for_connection() if node.conn: # if we can make a connection w/o a password then this is the # first pass set_timezone_info() setup_root_user(node.conn) create_db(node.conn) create_default_user(node.conn) create_repl_user(node.conn) run_external_scripts('/etc/initdb.d') expire_root_password(node.conn) else: ctx = dict(user=config.repl_user, password=config.repl_password, database=config.mysql_db) node.conn = wait_for_connection(**ctx) stop_replication(node.conn) # in case this is a newly-promoted primary if USE_STANDBY: # if we're using a standby instance then we need to first # snapshot the primary so that we can bootstrap the standby. write_snapshot(node.conn) Set up DB, user, replication user, and expire password, etc.
node.conn = wait_for_connection(**ctx) set_primary_for_replica(node.conn) except Exception as ex: log.exception(ex) def set_primary_for_replica(conn): """ Set up GTID-based replication to the primary; once this is set the replica will automatically try to catch up with the primary's last transactions. """ primary = get_primary_host() sql = ('CHANGE MASTER TO ' 'MASTER_HOST = %s, ' 'MASTER_USER = %s, ' 'MASTER_PASSWORD = %s, ' 'MASTER_PORT = 3306, ' 'MASTER_CONNECT_RETRY = 60, ' 'MASTER_AUTO_POSITION = 1, ' 'MASTER_SSL = 0; ' 'START SLAVE;') mysql_exec(conn, sql, (primary, config.repl_user, config.repl_password,))
node.conn = wait_for_connection(**ctx) set_primary_for_replica(node.conn) except Exception as ex: log.exception(ex) def set_primary_for_replica(conn): """ Set up GTID-based replication to the primary; once this is set the replica will automatically try to catch up with the primary's last transactions. """ primary = get_primary_host() sql = ('CHANGE MASTER TO ' 'MASTER_HOST = %s, ' 'MASTER_USER = %s, ' 'MASTER_PASSWORD = %s, ' 'MASTER_PORT = 3306, ' 'MASTER_CONNECT_RETRY = 60, ' 'MASTER_AUTO_POSITION = 1, ' 'MASTER_SSL = 0; ' 'START SLAVE;') mysql_exec(conn, sql, (primary, config.repl_user, config.repl_password,)) gets from Consul
node.conn = wait_for_connection(**ctx) set_primary_for_replica(node.conn) except Exception as ex: log.exception(ex) def set_primary_for_replica(conn): """ Set up GTID-based replication to the primary; once this is set the replica will automatically try to catch up with the primary's last transactions. """ primary = get_primary_host() sql = ('CHANGE MASTER TO ' 'MASTER_HOST = %s, ' 'MASTER_USER = %s, ' 'MASTER_PASSWORD = %s, ' 'MASTER_PORT = 3306, ' 'MASTER_CONNECT_RETRY = 60, ' 'MASTER_AUTO_POSITION = 1, ' 'MASTER_SSL = 0; ' 'START SLAVE;') mysql_exec(conn, sql, (primary, config.repl_user, config.repl_password,)) Remember our preStart downloaded the snapshot
Go back to start I’m the primary! Someone else is the primary! I’m a replica! Set lock in Consul w/ TTL Ask Consul for Primary Update lock TTL w/ each health check. Rewrite ContainerPilot config and SIGHUP
Also acts as a check for whether the ContainerPilot configuration needs to be reloaded (if it's been changed externally), or if we need to make a backup because the backup TTL has expired. """ node = MySQLNode() cp = ContainerPilot(node) if cp.update(): cp.reload() return was_ready = assert_initialized_for_state(node) # cp.reload() will exit early so no need to setup # connection until this point ctx = dict(user=config.repl_user, password=config.repl_password, timeout=cp.config['services'][0]['ttl']) node.conn = wait_for_connection(**ctx) # Update our lock on being the primary/standby. # If this lock is allowed to expire and the health check for the primary # fails, the `onChange` handlers for the replicas will try to self-elect # as primary by obtaining the lock. # If this node can update the lock but the DB fails its health check, # then the operator will need to manually intervene if they want to # force a failover. This architecture is a result of Consul not # permitting us to acquire a new lock on a health-checked session if the # health check is *currently* failing, but has the happy side-effect of # reducing the risk of flapping on a transient health check failure. if node.is_primary() or node.is_standby(): update_session_ttl() # Create a snapshot and send it to the object store. if all((node.is_snapshot_node(), (not is_backup_running()), (is_binlog_stale(node.conn) or is_time_for_snapshot()))): write_snapshot(node.conn) mysql_query(node.conn, 'SELECT 1', ())
password=config.repl_password, timeout=cp.config['services'][0]['ttl']) node.conn = wait_for_connection(**ctx) # need to stop replication whether we're the new primary or not stop_replication(node.conn) while True: try: # if there is no primary node, we'll try to obtain the lock. # if we get the lock we'll reload as the new primary, otherwise # someone else got the lock but we don't know who yet so loop primary = get_primary_node() if not primary: session_id = get_session(no_cache=True) if mark_with_session(PRIMARY_KEY, node.hostname, session_id): node.state = PRIMARY if cp.update(): cp.reload() return else: # we lost the race to lock the session for ourselves time.sleep(1) continue # we know who the primary is but not whether they're healthy. # if it's not healthy, we'll throw an exception and start over. ip = get_primary_host(primary=primary) if ip == node.ip: if cp.update(): cp.reload() return set_primary_for_replica(node.conn) return except Exception as ex: # This exception gets thrown if the session lock for `mysql-primary` # key has not expired yet (but there's no healthy primary either), # or if the replica's target primary isn't ready yet. log.debug(ex) time.sleep(1) # avoid hammering Consul continue
Ask Consul for Primary Ask Consul for Primary Ask Consul for Primary Ask Consul for Primary Ok, I’m primary Set lock in Consul Success! primary Healthy! fire onChange handler
tgross
sadnessojisan
coincheck_recruit
kaminashi
niwatakeru
tkikuchi
tenshoku_draft
kamina_zzz
eventhub
msato
akexorcist
bbrbbq
tohutohu
jakevdp
samlambert
bryan
malarkey
bluesmoon
kevinliebholz
shlominoach
csswizardry
danielanewman
sachag
hannesfritz
ammeep
![github.com/autopilotpattern “[a] pattern where containers autonomously adapt to changes in](https://files.speakerdeck.com/presentations/61337b95f48247a18f30cb958bc8ad19/slide_49.jpg){kind=link}
![github.com/autopilotpattern ~ $ git clone [email protected]:autopilotpattern/workshop.git ~ $ cd workshop](https://files.speakerdeck.com/presentations/61337b95f48247a18f30cb958bc8ad19/slide_84.jpg){kind=link}
![github.com/autopilotpattern var upstreamHosts = []; var getUpstreams = function(force, callback)](https://files.speakerdeck.com/presentations/61337b95f48247a18f30cb958bc8ad19/slide_128.jpg){kind=link}
![github.com/autopilotpattern ~ $ git clone [email protected]:autopilotpattern/mysql.git ~ $ cd mysql](https://files.speakerdeck.com/presentations/61337b95f48247a18f30cb958bc8ad19/slide_132.jpg){kind=link}
