Did you know JavaScript has iterators? DublinJS

How many ways do you know to do iteration with JavaScript and Node.js? While, for loop, for…in, for..of, .map(), .forEach(), streams, iterators, etc! Yes, there are a lot of ways! But did you know that JavaScript has iteration protocols to standardise synchronous and even asynchronous iteration? In this workshop we will learn about these protocols and discover how to build iterators and iterable objects, both synchronous and asynchronous. We will learn about some common use cases for these protocols, explore generators and async generators (great tools for iteration) and finally discuss some hot tips, common pitfalls, and some (more or less successful) wild ideas!

1. 2023-03-07 1
2. META_SLIDE! loige loige.link/iterate 2
3. String[] transactions = { "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" }; var total = Stream.of(transactions).mapToInt(transaction -> { var parts = transaction.split(" "); int amount = Integer.decode(parts[1]); if (Objects.equals(parts[0], "paid")) { amount = -amount; } return amount; }).sum(); if (total >= 0) { System.out.println("Life is good :)"); } else { System.out.println("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 loige replit.com/@lmammino/r-u-broke-java 3
4. String[] transactions = { "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" }; 1 2 3 4 5 6 7 8 9 var total = Stream.of(transactions).mapToInt(transaction -> { 10 var parts = transaction.split(" "); 11 int amount = Integer.decode(parts[1]); 12 if (Objects.equals(parts[0], "paid")) { 13 amount = -amount; 14 } 15 16 return amount; 17 }).sum(); 18 19 if (total >= 0) { 20 System.out.println("Life is good :)"); 21 } else { 22 System.out.println("You are broke :("); 23 } 24 var total = Stream.of(transactions).mapToInt(transaction -> { String[] transactions = { 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 }; 8 9 10 var parts = transaction.split(" "); 11 int amount = Integer.decode(parts[1]); 12 if (Objects.equals(parts[0], "paid")) { 13 amount = -amount; 14 } 15 16 return amount; 17 }).sum(); 18 19 if (total >= 0) { 20 System.out.println("Life is good :)"); 21 } else { 22 System.out.println("You are broke :("); 23 } 24 var parts = transaction.split(" "); int amount = Integer.decode(parts[1]); if (Objects.equals(parts[0], "paid")) { amount = -amount; } return amount; String[] transactions = { 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 }; 8 9 var total = Stream.of(transactions).mapToInt(transaction -> { 10 11 12 13 14 15 16 17 }).sum(); 18 19 if (total >= 0) { 20 System.out.println("Life is good :)"); 21 } else { 22 System.out.println("You are broke :("); 23 } 24 }).sum(); String[] transactions = { 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 }; 8 9 var total = Stream.of(transactions).mapToInt(transaction -> { 10 var parts = transaction.split(" "); 11 int amount = Integer.decode(parts[1]); 12 if (Objects.equals(parts[0], "paid")) { 13 amount = -amount; 14 } 15 16 return amount; 17 18 19 if (total >= 0) { 20 System.out.println("Life is good :)"); 21 } else { 22 System.out.println("You are broke :("); 23 } 24 if (total >= 0) { System.out.println("Life is good :)"); } else { System.out.println("You are broke :("); } String[] transactions = { 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 }; 8 9 var total = Stream.of(transactions).mapToInt(transaction -> { 10 var parts = transaction.split(" "); 11 int amount = Integer.decode(parts[1]); 12 if (Objects.equals(parts[0], "paid")) { 13 amount = -amount; 14 } 15 16 return amount; 17 }).sum(); 18 19 20 21 22 23 24 String[] transactions = { "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" }; var total = Stream.of(transactions).mapToInt(transaction -> { var parts = transaction.split(" "); int amount = Integer.decode(parts[1]); if (Objects.equals(parts[0], "paid")) { amount = -amount; } return amount; }).sum(); if (total >= 0) { System.out.println("Life is good :)"); } else { System.out.println("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 loige replit.com/@lmammino/r-u-broke-java 4
5. transactions = [ "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" ]; def get_amount(transaction): type, amount = transaction.split(' ') amount = int(amount) if type == 'paid': return -amount return amount *_, total = accumulate( map(get_amount, transactions) ) if total >= 0: print("Life is good :)") else: print("You are broke :(") 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 loige replit.com/@lmammino/r-u-broke-py 5
6. let transactions = [ "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12", ]; let total: i32 = transactions .iter() .map(|transaction| { let (action, amount) = transaction.split_once(' ').unwrap(); let amount = amount.parse::<i32>().unwrap(); if action == "paid" { -amount } else { amount } }) .sum(); if total >= 0 { println!("Life is good :)"); } else { println!("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 loige replit.com/@lmammino/r-u-broke-rust 6
7. const transactions = [ "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" ] const total = Iterator.from(transactions) .map(transaction => { let [action, amount] = transaction.split(" ") amount = Number.parseInt(amount) if (action === "paid") { return -amount } else { return amount } }) .reduce((acc,curr) => acc + curr) if (total >= 0) { console.log("Life is good :)") } else { console.log("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 loige replit.com/@lmammino/r-u-broke-js 7
8. const transactions = [ "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" ] const total = Iterator.from(transactions) .map(transaction => { let [action, amount] = transaction.split(" ") amount = Number.parseInt(amount) if (action === "paid") { return -amount } else { return amount } }) .reduce((acc,curr) => acc + curr) if (total >= 0) { console.log("Life is good :)") } else { console.log("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 const total = Iterator.from(transactions) const transactions = [ 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 ] 8 9 10 .map(transaction => { 11 let [action, amount] = transaction.split(" ") 12 amount = Number.parseInt(amount) 13 if (action === "paid") { 14 return -amount 15 } else { 16 return amount 17 } 18 }) 19 .reduce((acc,curr) => acc + curr) 20 21 if (total >= 0) { 22 console.log("Life is good :)") 23 } else { 24 console.log("You are broke :("); 25 } 26 .map(transaction => { const transactions = [ 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 ] 8 9 const total = Iterator.from(transactions) 10 11 let [action, amount] = transaction.split(" ") 12 amount = Number.parseInt(amount) 13 if (action === "paid") { 14 return -amount 15 } else { 16 return amount 17 } 18 }) 19 .reduce((acc,curr) => acc + curr) 20 21 if (total >= 0) { 22 console.log("Life is good :)") 23 } else { 24 console.log("You are broke :("); 25 } 26 .reduce((acc,curr) => acc + curr) const transactions = [ 1 "paid 20", 2 "received 10", 3 "paid 5", 4 "received 15", 5 "paid 10", 6 "received 12" 7 ] 8 9 const total = Iterator.from(transactions) 10 .map(transaction => { 11 let [action, amount] = transaction.split(" ") 12 amount = Number.parseInt(amount) 13 if (action === "paid") { 14 return -amount 15 } else { 16 return amount 17 } 18 }) 19 20 21 if (total >= 0) { 22 console.log("Life is good :)") 23 } else { 24 console.log("You are broke :("); 25 } 26 const transactions = [ "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" ] const total = Iterator.from(transactions) .map(transaction => { let [action, amount] = transaction.split(" ") amount = Number.parseInt(amount) if (action === "paid") { return -amount } else { return amount } }) .reduce((acc,curr) => acc + curr) if (total >= 0) { console.log("Life is good :)") } else { console.log("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 loige replit.com/@lmammino/r-u-broke-js 8
9. const transactions = [ "paid 20", "received 10", "paid 5", "received 15", "paid 10", "received 12" ] const total = Iterator.from(transactions) .map(transaction => { let [action, amount] = transaction.split(" ") amount = Number.parseInt(amount) if (action === "paid") { return -amount } else { return amount } }) .reduce((acc,curr) => acc + curr) if (total >= 0) { console.log("Life is good :)") } else { console.log("You are broke :("); } 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 loige replit.com/@lmammino/r-u-broke-js This is not JavaScript, it's FutureJavaScript™ github.com/tc39/proposal-iterator-helpers 9
10. import Iterator from 'core-js-pure/actual/iterator/index.js' 1 2 const transactions = [ 3 "paid 20", 4 "received 10", 5 "paid 5", 6 "received 15", 7 "paid 10", 8 "received 12" 9 ] 10 11 const total = Iterator.from(transactions) 12 .map(transaction => { 13 let [action, amount] = transaction.split(" ") 14 amount = Number.parseInt(amount) 15 if (action === "paid") { 16 return -amount 17 } else { 18 return amount 19 } 20 }) 21 .reduce((acc,curr) => acc + curr) 22 23 if (total >= 0) { 24 console.log("Life is good :)") 25 } else { 26 console.log("You are broke :("); 27 loige But if you want the future, today... npm i --save core-js-pure 10
11. loige WHY? 11
12. loige ITERATORS ARE LAZY! You can consume the collection 1 item at the time You don't need to keep all the items in memory Great for large datasets You can even have endless iterators! 12
13. loige The concept of Iterators exists already in JavaScript... 🗞 GOOD NEWS FROM THE WORLD... ... since ES2015! 13
14. WAIT, WHO THE HECK IS THIS GUY !? 🤷 👋 I'm Luciano ( 🍕🍝) Senior Architect @ fourTheorem (Dublin ) nodejsdp.link 📔 Co-Author of Node.js Design Patterns 👉 Let's connect! (blog) (twitter) (twitch) (github) loige.co @loige loige lmammino 14
15. ALWAYS RE-IMAGINING WE ARE A PIONEERING TECHNOLOGY CONSULTANCY FOCUSED ON CLOUD, AWS & SERVERLESS loige 😇 We are always looking for talent: fth.link/careers We can help with: Cloud Migrations Training & Cloud enablement Building serverless applications Cutting cloud costs 15
16. loige podcast awsbites.com IF YOU LIKE AWS... 16
17. loige 🥤 LET'S HAVE A QUICK REFRESHER ON SOME JS "ITERATION" CONCEPTS... 17
18. const array = ['foo', 'bar', 'baz'] for (const item of array) { console.log(item) } loige Output: foo bar baz 18
19. const str = 'foo' for (const item of str) { console.log(item) } loige Output: f o o 19
20. const set = new Set(['foo', 'bar', 'baz']) for (const item of set) { console.log(item) } loige Output: foo bar baz 20
21. const map = new Map([ ['foo', 'bar'], ['baz', 'qux'] ]) for (const item of map) { console.log(item) } loige Output: [ 'foo', 'bar' ] [ 'baz', 'qux' ] 21
22. const obj = { foo: 'bar', baz: 'qux' } for (const item of obj) { console.log(item) } loige Output: ⛔ Uncaught TypeError: obj is not iterable OMG `for ... of` does not work with plain objects! 😱 22
23. const obj = { foo: 'bar', baz: 'qux' } for (const item of Object.entries(obj)) { console.log(item) } loige Output: [ 'foo', 'bar' ] [ 'baz', 'qux' ] 23
24. loige 24
25. const array = ['foo', 'bar', 'baz'] console.log(...array) loige Output: foo bar baz spread syntax! 25
26. loige 26
27. loige 27
28. 📒 AGENDA Generators Iterator protocol Iterable protocol Async Iterator protocol Async Iterable protcol Real-life™ examples loige 28
29. ➡ GENERATORS loige 29
30. GENERATOR FN & OBJ loige function * myGenerator () { // generator body yield 'someValue' // ... do more stuff } const genObj = myGenerator() genObj.next() // -> { done: false, value: 'someValue' } 30
31. function * fruitGen () { yield ' 🍑 ' yield ' 🍉 ' yield ' 🍋 ' yield ' 🥭 ' } const fruitGenObj = fruitGen() console.log(fruitGenObj.next()) // { value: ' 🍑 ', done: false } console.log(fruitGenObj.next()) // { value: ' 🍉 ', done: false } console.log(fruitGenObj.next()) // { value: ' 🍋 ', done: false } console.log(fruitGenObj.next()) // { value: ' 🥭 ', done: false } console.log(fruitGenObj.next()) // { value: undefined, done: true } loige 31
32. function * fruitGen () { yield ' 🍑 ' yield ' 🍉 ' yield ' 🍋 ' yield ' 🥭 ' } const fruitGenObj = fruitGen() // generator objects are iterable! for (const fruit of fruitGenObj) { console.log(fruit) } // 🍑 // 🍉 // 🍋 // 🥭 loige 32
33. function * range (start, end) { for (let i = start; i < end; i++) { yield i } } // generators are lazy! for (const i of range(0, Number.MAX_VALUE)) { console.log(i) } const zeroToTen = [...range(0, 11)] loige 33
34. // generators can be "endless" function * cycle (values) { let current = 0 while (true) { yield values[current] current = (current + 1) % values.length } } for (const value of cycle(['even', 'odd'])) { console.log(value) } // even // odd // even // ... loige 34
35. 📝 MINI-SUMMARY loige A generator function returns a generator object which is both an iterator and an iterable. A generator function uses `yield` to yield a value and pause its execution. The generator object is used to make progress on an instance of the generator (by calling `next()`). Generator functions are a great way to create custom iterable objects. Generator objects are lazy and they can be endless. 35
36. ➡ ITERATORS & ITERABLES loige 36
37. ITERATOR OBJ loige An object that acts like a cursor to iterate over blocks of data sequentially 37
38. ITERABLE OBJ loige An object that contains data that can be iterated over sequentially 38
39. THE ITERATOR PROTOCOL An object is an iterator if it has a next() method. Every time you call it, it returns an object with the keys done (boolean) and value. loige 39
40. function createCountdown (from) { let nextVal = from return { next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 40
41. const countdown = createCountdown(3) console.log(countdown.next()) // { done: false, value: 3 } console.log(countdown.next()) // { done: false, value: 2 } console.log(countdown.next()) // { done: false, value: 1 } console.log(countdown.next()) // { done: false, value: 0 } console.log(countdown.next()) // { done: true } loige 41
42. 🔥 Generator Objects are iterators (and iterables)! loige 42
43. function * createCountdown (from) { for (let i = from; i >= 0; i--) { yield i } } loige 43
44. const countdown = createCountdown(3) console.log(countdown.next()) // { done: false, value: 3 } console.log(countdown.next()) // { done: false, value: 2 } console.log(countdown.next()) // { done: false, value: 1 } console.log(countdown.next()) // { done: false, value: 0 } console.log(countdown.next()) // { done: true, value: undefined } loige 44
45. THE ITERABLE PROTOCOL An object is iterable if it implements the Symbol.iterator method, a zero-argument function that returns an iterator. loige 45
46. function createCountdown (from) { let nextVal = from return { [Symbol.iterator]: () => ({ next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } }) } } loige 46
47. function createCountdown (from) { return { [Symbol.iterator]: function * () { for (let i = from; i >= 0; i--) { yield i } } } } loige 47
48. function * createCountdown () { for (let i = from; i >= 0; i--) { yield i } } loige 🔥 or just use generators! 48
49. const countdown = createCountdown(3) for (const value of countdown) { console.log(value) } // 3 // 2 // 1 // 0 loige 49
50. const iterableIterator = { next () { return { done: false, value: 'hello' } }, [Symbol.iterator] () { return this } } An object can be an iterable and an iterator at the same time! loige 50
51. 📝 MINI-SUMMARY 1/2 loige An iterator is an object that allows us to traverse a collection The iterator protocol specifies that an object is an iterator if it has a `next()` method that returns an object with the shape `{done, value}`. `done` (a boolean) tells us if the iteration is completed `value` represents the value from the current iteration. You can write an iterator as an anonymous object (e.g. returned by a factory function), using classes or using generators. 51
52. 📝 MINI-SUMMARY 2/2 loige The iterable protocol defines what's expected for a JavaScript object to be considered iterable. That is an object that holds a collection of data on which you can iterate on sequentially. An object is iterable if it implements a special method called `Symbol.iterator` which returns an iterator. (An object is iterable if you can get an iterator from it!) Generator functions produce objects that are iterable. We saw that generators produce objects that are also iterators. It is possible to have objects that are both iterator and iterable. The trick is to create the object as an iterator and to implement a `Symbol.iterator` that returns the object itself (`this`). 52
53. OK, very cool! But, so far this is all synchronous iteration. What about async? 🙄 loige 53
54. ➡ ASYNC ITERATORS & ITERABLES loige 54
55. THE ASYNC ITERATOR PROTOCOL An object is an async iterator if it has a next() method. Every time you call it, it returns a promise that resolves to an object with the keys done (boolean) and value. loige 55
56. import { setTimeout } from 'node:timers/promises' function createAsyncCountdown (from, delay = 1000) { let nextVal = from return { async next () { await setTimeout(delay) if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 56
57. const countdown = createAsyncCountdown(3) console.log(await countdown.next()) // { done: false, value: 3 } console.log(await countdown.next()) // { done: false, value: 2 } console.log(await countdown.next()) // { done: false, value: 1 } console.log(await countdown.next()) // { done: false, value: 0 } console.log(await countdown.next()) // { done: true } loige 57
58. loige 58
59. import { setTimeout } from 'node:timers/promises' // async generators "produce" async iterators! async function * createAsyncCountdown (from, delay = 1000) { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } loige 59
60. THE ASYNC ITERABLE PROTOCOL An object is an async iterable if it implements the `Symbol.asyncIterator` method, a zero-argument function that returns an async iterator. loige 60
61. import { setTimeout } from 'node:timers/promises' function createAsyncCountdown (from, delay = 1000) { return { [Symbol.asyncIterator]: async function * () { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } } } loige 61
62. HOT TIP 🔥 With async generators we can create objects that are both async iterators and async iterables! (We don't need to specify Symbol.asyncIterator explicitly!) loige 62
63. import { setTimeout } from 'node:timers/promises' // async generators "produce" async iterators // (and iterables!) async function * createAsyncCountdown (from, delay = 1000) { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } loige 63
64. const countdown = createAsyncCountdown(3) for await (const value of countdown) { console.log(value) } loige 64
65. 📝 MINI-SUMMARY 1/2 loige Async iterators are the asynchronous counterpart of iterators. They are useful to iterate over data that becomes available asynchronously (e.g. coming from a database or a REST API). A good example is a paginated API, we could build an async iterator that gives a new page for every iteration. An object is an async iterator if it has a `next()` method which returns a `Promise` that resolves to an object with the shape: `{done, value}`. The main difference with the iterator protocol is that this time `next()` returns a promise. When we call next we need to make sure we `await` the returned promise. 65
66. 📝 MINI-SUMMARY 2/2 loige The async iterable protocol defines what it means for an object to be an async iterable. Once you have an async iterable you can use the `for await ... of` syntax on it. An object is an async iterable if it has a special method called `Symbol.asyncIterator` that returns an async iterator. Async iterables are a great way to abstract paginated data that is available asynchronously or similar operations like pulling jobs from a remote queue. A small spoiler, async iterables can also be used with Node.js streams... 66
67. A REAL USE CASE & A CHALLENGE! loige Async Iterables are commonly used to abstract over paginated resources. Can you implement a paginator for the Rick & Morty API? function createCharactersPaginator () { // return an async iterable object that produces // pages with name of Rick and Morty characters // taken from the API // https://rickandmortyapi.com/api/character } // This is what we want to support 👇 const paginator = createCharactersPaginator() for await (const page of paginator) { console.log(page) } 67
68. IN CONCLUSION 💥 loige JavaScript has built-in sync/async iteratable/iterators These are great tools when you need to deal with large (or even endless) data sets They give you great control over how to consume data incrementally Iterator helpers are still lacking and not as mature as in other languages ... although, you should know enough to be able to create your own helpers now 😜 ... and you can leverage generator functions to make your life easier! ... or you can wait for & to land Iterator Helpers Async Iterator Helpers ... or you can use to polyfill these features today! core-js 68
69. BONUS: A FREE WORKSHOP FOR YOU! 🎁 loige loige.link/lets-iter-repo git clone https://github.com/lmammino/iteration-protocols-workshop.git cd iteration-protocols-workshop npm i 69
70. Front cover Photo by on Back cover photo by on Sam Barber Unsplash photonblast Unsplash fourtheorem.com THANKS! 🙌 loige nodejsdp.link 70