Observables - Streams on steroids

Asynchronous control flows built one of the foundations for Node.js's widespread success. Tracing the execution path through all the callbacks increased the cognitive load, though.

Let Stephan Schneider show you a way out of async batch processing utilizing "Reactive Extensions" to apply multiple steps of business logic on each item - whenever it flows in. The talk will include the TC39 proposal for Observables!

Transcript

1. Observables Streams on steroids @zcei_GER Stephan Schneider
2. 2 What is an Observable?
3. 3 IT’S A FUNCTION
4. function countToThree () { return new Observable((observer) => { observer.next(1) observer.next(2) observer.next(3) observer.complete() }) } const doubledValues = countToThree().map((value) => value * 2) OBSERVABLE How to create one? Here it is This is one as well
5. const observer = { next(value) { console.log('next:', value) }, error(err) { console.error(err) }, complete() { console.log('We are done') }, } const subscription = doubledValues.subscribe(observer) // next: 2 // next: 4 // next: 6 // We are done OBSERVER / SUBSCRIPTION How do I use it? Starts data emission
6. Whenever you model a flow that operates on individual items MESSAGES EVENTS •AMQP •Kafka •RPC •EventEmitter •DOMEvent •Streams WHEN SHOULD I USE IT?
7. HOW DOES IT COMPARE TO STREAMS? Single Operator • chunk • encoding Network / File Nomenclature • pipe Abstract Interface • _read • _write STREAM OBSERVABLE No Specific Nomenclature • value • n/a Infinite Operators • map • count • … Minimal Interface • subscribe • next
8. // const map = require('through2-map') function countToThree () { const rs = new Readable({ objectMode: true }) let count = 0 rs._read = () => { if (c === 3) { rs.push(null) } count = count + 1 rs.push(count) } } const doubledValues = countToThree().pipe(map((value) => value * 2)) doubledValues.pipe(process.stdout) STREAMS Idiomatic implementation
9. // const map = require('through2-map') function countToThree () { const rs = new Readable({ objectMode: true }) let count = 0 rs._read = () => { if (c === 3) { rs.push(null) } count = count + 1 rs.push(count) } } const doubledValues = countToThree().pipe(map((value) => value * 2)) doubledValues.pipe(process.stdout) STREAMS Same functionality as before Oh oh, no automatic newline
10. // const map = require('through2-map') function countToThree () { const rs = new Readable({ objectMode: true }) let count = 0 rs._read = () => { if (c === 3) { rs.push(null) } count = count + 1 rs.push(count) } } const doubledValues = countToThree().pipe(map((value) => `${value * 2}\n`)) doubledValues.pipe(process.stdout) STREAMS Same functionality as before Complete on null value Ugly Implement interface method
11. LIST FUNCTIONALS filter, map, reduce: count errors in log file scan: current shopping cart value
12. filter, map, reduce: count errors in log file scan: current shopping cart value VIDEO!
13. const readStream = fs .createReadStream(LOG_FILE, { encoding: 'utf8' }) .pipe(split('\n')) const logAggregation = streamToObservable(readStream) .filter((line) => Boolean(line)) .map((line) => JSON.parse(line)) .reduce(countByType, { errors: 0, warnings: 0 }) logAggregation.subscribe(logResult) MAP, FILTER, REDUCE Combine to a single value
14. const readStream = fs .createReadStream(LOG_FILE, { encoding: 'utf8' }) .pipe(split('\n')) const logAggregation = streamToObservable(readStream) .filter((line) => Boolean(line)) .map((line) => JSON.parse(line)) .reduce(countByType, { errors: 0, warnings: 0 }) logAggregation.subscribe(logResult) MAP, FILTER, REDUCE Combine to a single value filter empty lines in log file transforms only items which satisfied filter fn called exactly once
15. filter, map, reduce: count errors in log file scan: current shopping cart value VIDEO!
16. function observableShell () { return new Observable(function (observer) { const list = // ... list.on('change', (value) => observer.next(value)) list.on('select', () => observer.complete()) return () => list.removeAllListeners() }) } const selection = Observable .from(observableShell()) .scan(collectSelections, []) .map(cartText) SCAN Reduce with intermediary results
17. function observableShell () { return new Observable(function (observer) { const list = // ... list.on('change', (value) => observer.next(value)) list.on('select', () => observer.complete()) return () => list.removeAllListeners() }) } const selection = Observable .from(observableShell()) .scan(collectSelections, []) .map(cartText) SCAN Reduce with intermediary results reduce items into an Array is called with current array for each item The item we (un)selected
18. MERGE OBSERVABLES merge: newsfeed from different sources mergeMap: fetch weather for each city
19. filter, map, reduce: count errors in log file scan: current shopping cart value VIDEO!
20. function feed (source, frequency) { return new Rx.Observable(function (observer) { const interval = setInterval(() => { observer.next(randomMessage(source)) }, frequency) return () => clearInterval(interval) }) } const twitterFeed = feed('Twitter', 200) const hnFeed = feed('Hacker News', 500) const newsfeed = Rx.Observable.merge(twitterFeed, hnFeed) newsfeed.subscribe(logResult) MERGE Combine two Observables into one
21. function feed (source, frequency) { return new Rx.Observable(function (observer) { const interval = setInterval(() => { observer.next(randomMessage(source)) }, frequency) return () => clearInterval(interval) }) } const twitterFeed = feed('Twitter', 200) const hnFeed = feed('Hacker News', 500) const newsfeed = Rx.Observable.merge(twitterFeed, hnFeed) newsfeed.subscribe(logResult) MERGE Combine two Observables into one Cleanup merge both Observables into a new one
22. filter, map, reduce: count errors in log file scan: current shopping cart value VIDEO!
23. function getWeatherForCity (city) { return Rx.Observable .from(fetch(city)) .map((response) => response.weather) .map((weather) => ({ city, weather })) } const weatherFeed = Rx.Observable .from(['Berlin', 'London', 'New York']) .mergeMap(getWeatherForCity) weatherFeed.subscribe(logResult) MERGE MAP transform an element to an observable
24. function getWeatherForCity (city) { return Rx.Observable .from(fetch(city)) .map((response) => response.weather) .map((weather) => ({ city, weather })) } const weatherFeed = Rx.Observable .from(['Berlin', 'London', 'New York']) .mergeMap(getWeatherForCity) weatherFeed.subscribe(logResult) MERGE MAP transform an element to an observable `fetch` returns a Promise extract / transform data merge all items from this observable into ours
25. SEQUENTIAL PROCESSING concat: real-estate list with promoted items first
26. filter, map, reduce: count errors in log file scan: current shopping cart value VIDEO!
27. function asyncListObservable (asyncList, ...args) { return Rx.Observable .from(asyncList(...args)) .mergeMap((list) => Rx.Observable.of(...list)) } const promoted = asyncListObservable(fetch, 'promoted') const normal = asyncListObservable(fetch) const realEstateFeed = Rx.Observable.concat(promoted, normal) realEstateFeed.subscribe(logResult) CONCAT Combine two Observables sequentially
28. function asyncListObservable (asyncList, ...args) { return Rx.Observable .from(asyncList(...args)) .mergeMap((list) => Rx.Observable.of(...list)) } const promoted = asyncListObservable(fetch, 'promoted') const normal = asyncListObservable(fetch) const realEstateFeed = Rx.Observable.concat(promoted, normal) realEstateFeed.subscribe(logResult) CONCAT Combine two Observables sequentially Emit each Array value wait for first Observable to finish
29. 29 What are Observables? CONTROL LIGHTWEIGHT CUSTOMISABLE FOUNDER
30. 30 Are Observables the solution? ¯\_(ツ)_/¯
31. Thanks! @zcei_GER provided to Node Meetup Group Slides