Real Time Web with Node

This was my node talk at TXJS 2009. It announces the Connect framework and shows how to use it to make real-time apps with node.

Transcript

1. REAL TIME WEB WITH NODE By Tim Caswell Saturday, June 5, 2010
2. Live Interaction The web is about doing things, not just tourism anymore. Saturday, June 5, 2010
3. Chat Widget Twitter Feed Stock Ticker Real-Time Game Collaborative Documents TXJS Demos Uses of Live Interaction Saturday, June 5, 2010
4. Why we need non-blocking Polling is too slow and inefficient. Live interaction requires the server to push data. In order to push data, the connections need to be persistent. The server needs to handle thousands of persistent connections Threads aren’t going to scale. Saturday, June 5, 2010
5. This is your server. (with blocking I/O) Saturday, June 5, 2010
6. This is your server with low concurrency. Saturday, June 5, 2010
7. This is your server with high concurrency. Saturday, June 5, 2010
8. Connect We’ll use a new node framework that “connects” the web users to each other. Saturday, June 5, 2010
9. It’s like Japanese Legos Saturday, June 5, 2010
10. Connect.createServer([ {filter: "log"}, {filter: "body-decoder", route: "/stream"}, {provider: "pubsub", route: "/stream"}, {filter: "conditional-get"}, {filter: "cache"}, {filter: "gzip"}, {provider: "cache-manifest", root: root}, {provider: "static", root: root} ]); Pre-Built Blocks Saturday, June 5, 2010
11. And easy too! Saturday, June 5, 2010
12. method-override.js var key; // Initialize any state (on server startup) exports.setup = function (env) { key = this.key || "_method"; }; // Modify the request stream (on request) exports.handle = function(err, req, res, next){ if (key in req.body) { req.method = req.body[key].toUpperCase(); } next(); }; Saturday, June 5, 2010
13. response-time.js exports.handle = function(err, req, res, next){ var start = new Date, writeHead = res.writeHead; res.writeHead = function(code, headers){ res.writeHead = writeHead; headers['X-Response-Time'] = (new Date - start) + "ms"; res.writeHead(code, headers); }; next(); }; Saturday, June 5, 2010
14. Well, actually, it’s not always easy. Saturday, June 5, 2010
15. static.js Saturday, June 5, 2010
16. static.js var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
17. static.js // Buffer any events that fire while waiting on the stat. var events = []; function onData() { events.push(["data"].concat(Array.prototype.slice.call(arguments))); } function onEnd() { events.push(["end"].concat(Array.prototype.slice.call(arguments))); } req.addListener("data", onData); req.addListener("end", onEnd); fs.stat(filename, function (err, stat) { // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); // Fall through for missing files, thow error for other problems if (err) { if (err.errno === process.ENOENT) { next(); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); }); return; var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
18. static.js // Buffer any events that fire while waiting on the stat. var events = []; function onData() { events.push(["data"].concat(Array.prototype.slice.call(arguments))); } function onEnd() { events.push(["end"].concat(Array.prototype.slice.call(arguments))); } req.addListener("data", onData); req.addListener("end", onEnd); fs.stat(filename, function (err, stat) { // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); // Fall through for missing files, thow error for other problems if (err) { if (err.errno === process.ENOENT) { next(); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); }); return; (err); rn; the file directly using buffers ile(filename, function (err, data) { err) { next(err); return; writeHead(200, { ontent-Type": Mime.type(filename), ontent-Length": data.length, ast-Modified": stat.mtime.toUTCString(), / Cache in browser for 1 year ache-Control": "public max-age=" + 31536000 end(data); var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
19. static.js // Buffer any events that fire while waiting on the stat. var events = []; function onData() { events.push(["data"].concat(Array.prototype.slice.call(arguments))); } function onEnd() { events.push(["end"].concat(Array.prototype.slice.call(arguments))); } req.addListener("data", onData); req.addListener("end", onEnd); fs.stat(filename, function (err, stat) { // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); // Fall through for missing files, thow error for other problems if (err) { if (err.errno === process.ENOENT) { next(); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); }); return; (err); rn; the file directly using buffers ile(filename, function (err, data) { err) { next(err); return; writeHead(200, { ontent-Type": Mime.type(filename), ontent-Length": data.length, ast-Modified": stat.mtime.toUTCString(), / Cache in browser for 1 year ache-Control": "public max-age=" + 31536000 end(data); }; // Mini mime module for static file serving var Mime = { type: function getMime(path) { var index = path.lastIndexOf("."); if (index < 0) { return DEFAULT_MIME; } var type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; return (/(text|javascript)/).test(type) ? type + "; charset=utf-8" : type; }, TYPES : { ".3gp" : "video/3gpp", ".a" : "application/octet-stream", ".ai" : "application/postscript", ".aif" : "audio/x-aiff", ".aiff" : "audio/x-aiff", ".asc" : "application/pgp-signature", ".asf" : "video/x-ms-asf", ".asm" : "text/x-asm", ".asx" : "video/x-ms-asf", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".avi" : "video/x-msvideo", var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
20. static.js ".flv" : "video/x-flv", ".for" : "text/x-fortran", ".gem" : "application/octet-stream", ".gemspec" : "text/x-script.ruby", ".gif" : "image/gif", ".gz" : "application/x-gzip", ".h" : "text/x-c", ".hh" : "text/x-c", ".htm" : "text/html", ".html" : "text/html", ".ico" : "image/vnd.microsoft.icon", ".ics" : "text/calendar", ".ifb" : "text/calendar", ".iso" : "application/octet-stream", ".jar" : "application/java-archive", ".java" : "text/x-java-source", ".jnlp" : "application/x-java-jnlp-file", ".jpeg" : "image/jpeg", ".jpg" : "image/jpeg", ".js" : "application/javascript", ".json" : "application/json", ".log" : "text/plain", ".m3u" : "audio/x-mpegurl", ".m4v" : "video/mp4", ".man" : "text/troff", ".mathml" : "application/mathml+xml", ".mbox" : "application/mbox", ".mdoc" : "text/troff", ".me" : "text/troff", ".mid" : "audio/midi", // Buffer any events that fire while waiting on the stat. var events = []; function onData() { events.push(["data"].concat(Array.prototype.slice.call(arguments))); } function onEnd() { events.push(["end"].concat(Array.prototype.slice.call(arguments))); } req.addListener("data", onData); req.addListener("end", onEnd); fs.stat(filename, function (err, stat) { // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); // Fall through for missing files, thow error for other problems if (err) { if (err.errno === process.ENOENT) { next(); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); }); return; (err); rn; the file directly using buffers ile(filename, function (err, data) { err) { next(err); return; writeHead(200, { ontent-Type": Mime.type(filename), ontent-Length": data.length, ast-Modified": stat.mtime.toUTCString(), / Cache in browser for 1 year ache-Control": "public max-age=" + 31536000 end(data); }; // Mini mime module for static file serving var Mime = { type: function getMime(path) { var index = path.lastIndexOf("."); if (index < 0) { return DEFAULT_MIME; } var type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; return (/(text|javascript)/).test(type) ? type + "; charset=utf-8" : type; }, TYPES : { ".3gp" : "video/3gpp", ".a" : "application/octet-stream", ".ai" : "application/postscript", ".aif" : "audio/x-aiff", ".aiff" : "audio/x-aiff", ".asc" : "application/pgp-signature", ".asf" : "video/x-ms-asf", ".asm" : "text/x-asm", ".asx" : "video/x-ms-asf", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".avi" : "video/x-msvideo", var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
21. static.js ".flv" : "video/x-flv", ".for" : "text/x-fortran", ".gem" : "application/octet-stream", ".gemspec" : "text/x-script.ruby", ".gif" : "image/gif", ".gz" : "application/x-gzip", ".h" : "text/x-c", ".hh" : "text/x-c", ".htm" : "text/html", ".html" : "text/html", ".ico" : "image/vnd.microsoft.icon", ".ics" : "text/calendar", ".ifb" : "text/calendar", ".iso" : "application/octet-stream", ".jar" : "application/java-archive", ".java" : "text/x-java-source", ".jnlp" : "application/x-java-jnlp-file", ".jpeg" : "image/jpeg", ".jpg" : "image/jpeg", ".js" : "application/javascript", ".json" : "application/json", ".log" : "text/plain", ".m3u" : "audio/x-mpegurl", ".m4v" : "video/mp4", ".man" : "text/troff", ".mathml" : "application/mathml+xml", ".mbox" : "application/mbox", ".mdoc" : "text/troff", ".me" : "text/troff", ".mid" : "audio/midi", // Buffer any events that fire while waiting on the stat. var events = []; function onData() { events.push(["data"].concat(Array.prototype.slice.call(arguments))); } function onEnd() { events.push(["end"].concat(Array.prototype.slice.call(arguments))); } req.addListener("data", onData); req.addListener("end", onEnd); fs.stat(filename, function (err, stat) { // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); // Fall through for missing files, thow error for other problems if (err) { if (err.errno === process.ENOENT) { next(); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); }); return; (err); rn; the file directly using buffers ile(filename, function (err, data) { err) { next(err); return; writeHead(200, { ontent-Type": Mime.type(filename), ontent-Length": data.length, ast-Modified": stat.mtime.toUTCString(), / Cache in browser for 1 year ache-Control": "public max-age=" + 31536000 end(data); ".cc" : "text/x-c", ".chm" : "application/vnd.ms-htmlhelp", ".class" : "application/octet-stream", ".com" : "application/x-msdownload", ".conf" : "text/plain", ".cpp" : "text/x-c", ".crt" : "application/x-x509-ca-cert", ".css" : "text/css", ".csv" : "text/csv", ".cxx" : "text/x-c", ".deb" : "application/x-debian-package", ".der" : "application/x-x509-ca-cert", ".diff" : "text/x-diff", ".djv" : "image/vnd.djvu", ".djvu" : "image/vnd.djvu", ".dll" : "application/x-msdownload", ".dmg" : "application/octet-stream", ".doc" : "application/msword", ".dot" : "application/msword", ".dtd" : "application/xml-dtd", ".dvi" : "application/x-dvi", ".ear" : "application/java-archive", ".eml" : "message/rfc822", ".eps" : "application/postscript", ".exe" : "application/x-msdownload", ".f" : "text/x-fortran", ".f77" : "text/x-fortran", ".f90" : "text/x-fortran", }; // Mini mime module for static file serving var Mime = { type: function getMime(path) { var index = path.lastIndexOf("."); if (index < 0) { return DEFAULT_MIME; } var type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; return (/(text|javascript)/).test(type) ? type + "; charset=utf-8" : type; }, TYPES : { ".3gp" : "video/3gpp", ".a" : "application/octet-stream", ".ai" : "application/postscript", ".aif" : "audio/x-aiff", ".aiff" : "audio/x-aiff", ".asc" : "application/pgp-signature", ".asf" : "video/x-ms-asf", ".asm" : "text/x-asm", ".asx" : "video/x-ms-asf", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".avi" : "video/x-msvideo", var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
22. static.js ".flv" : "video/x-flv", ".for" : "text/x-fortran", ".gem" : "application/octet-stream", ".gemspec" : "text/x-script.ruby", ".gif" : "image/gif", ".gz" : "application/x-gzip", ".h" : "text/x-c", ".hh" : "text/x-c", ".htm" : "text/html", ".html" : "text/html", ".ico" : "image/vnd.microsoft.icon", ".ics" : "text/calendar", ".ifb" : "text/calendar", ".iso" : "application/octet-stream", ".jar" : "application/java-archive", ".java" : "text/x-java-source", ".jnlp" : "application/x-java-jnlp-file", ".jpeg" : "image/jpeg", ".jpg" : "image/jpeg", ".js" : "application/javascript", ".json" : "application/json", ".log" : "text/plain", ".m3u" : "audio/x-mpegurl", ".m4v" : "video/mp4", ".man" : "text/troff", ".mathml" : "application/mathml+xml", ".mbox" : "application/mbox", ".mdoc" : "text/troff", ".me" : "text/troff", ".mid" : "audio/midi", // Buffer any events that fire while waiting on the stat. var events = []; function onData() { events.push(["data"].concat(Array.prototype.slice.call(arguments))); } function onEnd() { events.push(["end"].concat(Array.prototype.slice.call(arguments))); } req.addListener("data", onData); req.addListener("end", onEnd); fs.stat(filename, function (err, stat) { // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); // Fall through for missing files, thow error for other problems if (err) { if (err.errno === process.ENOENT) { next(); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); }); return; (err); rn; the file directly using buffers ile(filename, function (err, data) { err) { next(err); return; writeHead(200, { ontent-Type": Mime.type(filename), ontent-Length": data.length, ast-Modified": stat.mtime.toUTCString(), / Cache in browser for 1 year ache-Control": "public max-age=" + 31536000 end(data); ".bmp" : "image/bmp", ".bz2" : "application/x-bzip2", ".c" : "text/x-c", ".cab" : "application/vnd.ms-cab-compressed", ".cc" : "text/x-c", ".chm" : "application/vnd.ms-htmlhelp", ".class" : "application/octet-stream", ".com" : "application/x-msdownload", ".conf" : "text/plain", ".cpp" : "text/x-c", ".crt" : "application/x-x509-ca-cert", ".css" : "text/css", ".csv" : "text/csv", ".cxx" : "text/x-c", ".deb" : "application/x-debian-package", ".der" : "application/x-x509-ca-cert", ".diff" : "text/x-diff", ".djv" : "image/vnd.djvu", ".djvu" : "image/vnd.djvu", ".dll" : "application/x-msdownload", ".dmg" : "application/octet-stream", ".doc" : "application/msword", ".dot" : "application/msword", ".dtd" : "application/xml-dtd", ".dvi" : "application/x-dvi", ".ear" : "application/java-archive", ".eml" : "message/rfc822", ".eps" : "application/postscript", ".exe" : "application/x-msdownload", ".f" : "text/x-fortran", ".f77" : "text/x-fortran", ".f90" : "text/x-fortran", }; // Mini mime module for static file serving var Mime = { type: function getMime(path) { var index = path.lastIndexOf("."); if (index < 0) { return DEFAULT_MIME; } var type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; return (/(text|javascript)/).test(type) ? type + "; charset=utf-8" : type; }, TYPES : { ".3gp" : "video/3gpp", ".a" : "application/octet-stream", ".ai" : "application/postscript", ".aif" : "audio/x-aiff", ".aiff" : "audio/x-aiff", ".asc" : "application/pgp-signature", ".asf" : "video/x-ms-asf", ".asm" : "text/x-asm", ".asx" : "video/x-ms-asf", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".avi" : "video/x-msvideo", var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/\.\.+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010
23. Built-in Filter Modules Authentication Authorization Body Decoder Cache Conditional Get Debug Error Handler Gzip Log Method Override Response Time Session Saturday, June 5, 2010
24. Built-in Data Providers Static Rest Router PubSub Cache Manifest Direct JSON-RPC More... Saturday, June 5, 2010
25. Demo Time! Saturday, June 5, 2010
26. app.js (stack) require.paths.unshift("./lib"); var Connect = require('connect'); var root = __dirname + "/public"; module.exports = Connect.createServer([ {filter: "log"}, {filter: "body-decoder"}, {provider: "pubsub", route: "/stream", logic: Backend}, {filter: "conditional-get"}, {filter: "cache"}, {filter: "gzip"}, {provider: "cache-manifest", root: root}, {provider: "static", root: root} ]); Saturday, June 5, 2010
27. app.js (Backend) var Backend = { subscribe: function (subscriber) { if (subscribers.indexOf(subscriber) < 0) { subscribers.push(subscriber); } }, unsubscribe: function (subscriber) { var pos = subscribers.indexOf(subscriber); if (pos >= 0) { subscribers.slice(pos); } }, publish: function (message, callback) { subscribers.forEach(function (subscriber) { subscriber.send(message); }); callback(); } }; Saturday, June 5, 2010
28. http://github.com/extjs/connect http://twitter.com/creationix http://raphaeljs.com http://nodejs.org Saturday, June 5, 2010
29. Any Questions ? Saturday, June 5, 2010