Reactive Streams with Rx at JavaOne 2014

Learn how Netflix applies the Rx programming model to streams of data in finite, infinite and distributed infinite stream uses cases including a distributed system doing anomaly detection.

The presentation digs into the details of flow control and various approaches to achieve it including "reactive pull", a dynamic push/pull approach that composes backpressure into the streams.

Presented at JavaOne 2014: https://oracleus.activeevents.com/2014/connect/sessionDetail.ww?SESSION_ID=5749

Video: http://www.parleys.com/play/543f8d5be4b06e1184ae4106

More information can be found at http://reactivex.io

Transcript

1. Ben Christensen
   Developer – Edge Engineering at Netflix
   @benjchristensen
   !
   !
   !
   !
   !
   !
   http://techblog.netflix.com/
   JavaOne - September 2014
   Reactive Streams with Rx
   

   View Slide

2. View Slide

3. RxJava
   http://github.com/ReactiveX/RxJava
   http://reactivex.io
   

   View Slide

4. View Slide

5. Single Multiple
   Sync T getData() Iterable getData()
   Async Future getData() Observable getData()
   

   View Slide

6. public interface Observer {
   public abstract void onCompleted()
   public abstract void onError(Throwable e)
   public abstract void onNext(T t)
   }
   public class Observable {
   public static Observable create(OnSubscribe f)
   public Subscription subscribe(Observer super T> subscriber)
   }
   Lazy
   Cold or Hot
   Asynchronous
   Push
   *onNext
   (onError | onCompleted)?
   

   View Slide

7. public interface Observer {
   public abstract void onCompleted()
   public abstract void onError(Throwable e)
   public abstract void onNext(T t)
   }
   public class Observable {
   public static Observable create(OnSubscribe f)
   public Subscription subscribe(Observer super T> subscriber)
   }
   Lazy
   Cold or Hot
   Asynchronous (or Synchronous)
   Push (or Pull)
   *onNext
   (onError | onCompleted)?
   

   View Slide

8. public interface Observer {
   public abstract void onCompleted()
   public abstract void onError(Throwable e)
   public abstract void onNext(T t)
   }
   public class Observable {
   public static Observable create(OnSubscribe f)
   public Subscription subscribe(Observer super T> subscriber)
   }
   Lazy
   Cold or Hot
   Asynchronous (or Synchronous)
   Push (or Pull)
   *onNext
   (onError | onCompleted)?
   

   View Slide

9. public interface Subscriber implements Observer {
   public abstract void onCompleted()
   public abstract void onError(Throwable e)
   public abstract void onNext(T t)
   }
   public class Observable {
   public static Observable create(OnSubscribe f)
   public Subscription subscribe(Subscriber super T> subscriber)
   }
   Lazy
   Cold or Hot
   Asynchronous (or Synchronous)
   Push (or Pull)
   *onNext
   (onError | onCompleted)?
   

   View Slide

10. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

11. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

12. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

13. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

14. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    “reactive pull”
    dynamic push/pull
    for flow control (backpressure)
    

    View Slide

15. public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

16. public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

17. public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    

    View Slide

18. Observable.create(subscriber -> {
    subscriber.onNext("Hello world!");
    subscriber.onCompleted();
    }).forEach(System.out::println);
    synchronous
    single value
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

19. Observable.create(subscriber -> {
    subscriber.onNext("Hello");
    subscriber.onNext("world");
    subscriber.onNext("!");
    subscriber.onCompleted();
    }).forEach(System.out::println);
    synchronous
    multiple values
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

20. Observable.create(subscriber -> {
    try {
    subscriber.onNext(doSomething());
    subscriber.onCompleted();
    } catch (Throwable e) {
    subscriber.onError(e);
    }
    }).subscribeOn(Schedulers.io())
    .forEach(System.out::println);
    asynchronous
    single value
    error notification
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

21. Observable.create(subscriber -> {
    int i = 0;
    while (!subscriber.isUnsubscribed()) {
    subscriber.onNext(i++);
    }
    }).take(10).forEach(System.out::println);
    synchronous
    multiple values
    with unsubscribe
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

22. Observable.create(subscriber -> {
    AtomicInteger i = new AtomicInteger();
    AtomicLong requested = new AtomicLong();
    subscriber.setProducer(r -> {
    if (requested.getAndAdd(r) == 0) {
    do {
    if (subscriber.isUnsubscribed()) { break; }
    subscriber.onNext(i.incrementAndGet());
    } while (requested.decrementAndGet() > 0);
    }
    });
    }).observeOn(Schedulers.newThread())
    .take(10).forEach(System.out::println);
    synchronous
    multiple values
    flow control with
    “reactive pull”
    backpressure
    thread scheduling
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

23. Observable.create(subscriber -> {
    AtomicInteger i = new AtomicInteger();
    AtomicLong requested = new AtomicLong();
    subscriber.setProducer(r -> {
    if (requested.getAndAdd(r) == 0) {
    do {
    if (subscriber.isUnsubscribed()) { break; }
    subscriber.onNext(i.incrementAndGet());
    } while (requested.decrementAndGet() > 0);
    }
    });
    }).observeOn(Schedulers.newThread())
    .take(10).forEach(System.out::println);
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

24. Observable.from(iterable)
    .observeOn(Schedulers.newThread())
    .take(10).forEach(System.out::println);
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    

    View Slide

25. finite
    

    View Slide

26. infinite
    finite
    

    View Slide

27. finite
    infinite
    distributed
    

    View Slide

28. View Slide

29. View Slide

30. View Slide

31. View Slide

32. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

33. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

34. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

35.  Observable  b  =  Observable.flatMap({  T  t  -­‐>    
           Observable  r  =  ...  transform  t  ...  
           return  r;  
     })
    flatMap
    

    View Slide

36.  Observable  b  =  Observable.flatMap({  T  t  -­‐>    
           Observable  r  =  ...  transform  t  ...  
           return  r;  
     })
    flatMap
    

    View Slide

37. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

38. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

39. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

40. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

41. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

42. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

43. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

44. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

45. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

46.        Observable.zip(a,  b,  {  a,  b,  -­‐>    
               ...  operate  on  values  from  both  a  &  b  ...  
               return  [a,  b];  //  i.e.  return  tuple  
           })
    

    View Slide

47. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

48. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

49. View Slide

50. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

51. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

52. class  VideoService  {  
         def  Observable  getPersonalizedListOfMovies(userId);  
         def  Observable  getBookmark(userId,  videoId);  
         def  Observable  getRating(userId,  videoId);  
         def  Observable  getMetadata(videoId);  
    }
    an observable api:
    finite
    Treat everything like a stream
    

    View Slide

53. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    1
    2
    3
    4
    5
    6
    

    View Slide

54. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    3
    4
    5
    

    View Slide

55. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    

    View Slide

56. infinite
    

    View Slide

57. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }
    

    View Slide

58. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }
    

    View Slide

59. View Slide

60. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }
    

    View Slide

61. View Slide

62. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }
    

    View Slide

63. View Slide

64. data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}
    

    View Slide

65. data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}
    

    View Slide

66. data => {"rollingCountFallbackFailure":0,"rollingCountFallbackSuccess":
    0,"propertyValue_circuitBreakerRequestVolumeThreshold":"20","propertyValue_circuitBreakerForceOpen":false,"propertyValue_metricsRol
    lingStatisticalWindowInMilliseconds":"10000","latencyTotal_mean":32,"type":"HystrixCommand","rollingCountResponsesFromCache":
    0,"rollingCountTimeout":0,"propertyValue_executionIsolationStrategy":"SEMAPHORE","instanceId":"34567","rollingCountFailure":
    0,"rollingCountExceptionsThrown":0,"latencyExecute_mean":32,"isCircuitBreakerOpen":false,"errorCount":
    0,"rollingCountSemaphoreRejected":0,"latencyTotal":{"0":0,"25":0,"50":12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100":
    1560},"requestCount":1376,"rollingCountCollapsedRequests":94516,"rollingCountShortCircuited":0,"latencyExecute":{"0":0,"25":0,"50":
    12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100":
    1560},"propertyValue_circuitBreakerSleepWindowInMilliseconds":"5000","currentConcurrentExecutionCount":
    0,"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests":"10","errorPercentage":0,"rollingCountThreadPoolRejected":
    0,"propertyValue_circuitBreakerEnabled":true,"propertyValue_executionIsolationThreadInterruptOnTimeout":true,"propertyValue_request
    CacheEnabled":true,"rollingCountFallbackRejection":0,"propertyValue_requestLogEnabled":true,"rollingCountSuccess":
    1432,"propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests":"10","propertyValue_circuitBreakerErrorThresholdPercentage":"5
    0","propertyValue_circuitBreakerForceClosed":false,"name":"GetPredictions","reportingHosts":
    4,"propertyValue_executionIsolationThreadPoolKeyOverride":"null","propertyValue_executionIsolationThreadTimeoutInMilliseconds":"100
    0"}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}
    

    View Slide

67. Flow Control
    

    View Slide

68. Flow Control
    (backpressure)
    

    View Slide

69. no backpressure needed
    Observable.from(iterable).take(1000).map(i -> "value_" + i).subscribe(System.out::println);
    

    View Slide

70. Observable.from(iterable).take(1000).map(i -> "value_" + i).subscribe(System.out::println);
    no backpressure needed
    synchronous on same thread
    (no queueing)
    

    View Slide

71. Observable.from(iterable).take(1000).map(i -> "value_" + i)
    .observeOn(Schedulers.computation()).subscribe(System.out::println);
    backpressure needed
    

    View Slide

72. Observable.from(iterable).take(1000).map(i -> "value_" + i)
    .observeOn(Schedulers.computation()).subscribe(System.out::println);
    backpressure needed
    asynchronous
    (queueing)
    

    View Slide

73. Flow Control Options
    

    View Slide

74. Block
    (callstack blocking and/or park the thread)
    

    View Slide

75. Temporal Operators
    (batch or drop data using time)
    

    View Slide

76. Observable.range(1, 1000000).sample(10, TimeUnit.MILLISECONDS).forEach(System.out::println);
    110584
    242165
    544453
    942880
    

    View Slide

77. Observable.range(1, 1000000).throttleFirst(10, TimeUnit.MILLISECONDS).forEach(System.out::println);
    1
    55463
    163962
    308545
    457445
    592638
    751789
    897159
    

    View Slide

78. Observable.range(1, 1000000).debounce(10, TimeUnit.MILLISECONDS).forEach(System.out::println);
    1000000
    

    View Slide

79. Observable.range(1, 1000000).buffer(10, TimeUnit.MILLISECONDS)
    .toBlocking().forEach(list -> System.out.println("batch: " + list.size()));
    batch: 71141
    batch: 49488
    batch: 141147
    batch: 141432
    batch: 195920
    batch: 240462
    batch: 160410
    

    View Slide

80. View Slide

81. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    

    View Slide

82. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    

    View Slide

83. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    

    View Slide

84. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    

    View Slide

85. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    

    View Slide

86. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    https://gist.github.com/benjchristensen/e4524a308456f3c21c0b
    

    View Slide

87. Observable.range(1, 1000000).window(50, TimeUnit.MILLISECONDS)
    .flatMap(window -> window.count())
    .toBlocking().forEach(count -> System.out.println("num items: " + count));
    num items: 477769
    num items: 155463
    num items: 366768
    

    View Slide

88. Observable.range(1, 1000000).window(500000)
    .flatMap(window -> window.count())
    .toBlocking().forEach(count -> System.out.println("num items: " + count));
    num items: 500000
    num items: 500000
    

    View Slide

89. Reactive Pull
    (dynamic push-pull)
    

    View Slide

90. Push when consumer
    keeps up with producer.
    !
    Switch to Pull
    when consumer is slow.
    !
    Bound all* queues.
    

    View Slide

91. Push when consumer
    keeps up with producer.
    !
    Switch to Pull
    when consumer is slow.
    !
    Bound all* queues.
    *vertically, not horizontally
    

    View Slide

92. Observable Subscriber
    Observable.create(subscriber -> {
    // emit to subscriber
    }).subscribe(new Subscriber() {
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    

    View Slide

93. observable.subscribe
    Observable.create(subscriber -> {
    // emit to subscriber
    }).subscribe(new Subscriber() {
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    

    View Slide

94. observable.subscribe
    subscriber.setProducer
    Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    });
    }).subscribe(new Subscriber() {
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    

    View Slide

95. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    

    View Slide

96. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    subscriber.onNext(t); // 5 times
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    subscriber.onNext(t) …
    

    View Slide

97. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    subscriber.onNext(t); // 5 times
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    doStuffWith(t);
    if(requested—- == 0) request(5);
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    subscriber.onNext(t) …
    producer.request(n)
    

    View Slide

98. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    … previous logic …
    subscriber.onCompleted();
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    doStuffWith(t);
    if(requested—- == 0) request(5);
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    subscriber.onNext(t) …
    producer.request(n)
    subscriber.onCompleted/onError
    unsubscribe
    

    View Slide

99. Observable Subscriber
    Observable Subscriber
    observable.subscribe
    observable.subscribe
    

    View Slide

100. Observable.from(iterable) .observeOn(Schedulers.computation())
     .subscribe(System.out::println);
     

     View Slide

101. Observable.from(iterable) .observeOn(Schedulers.computation())
     .subscribe(System.out::println);
     

     View Slide

102. Observable Subscriber
     Observable Subscriber
     observable.subscribe
     observable.subscribe
     

     View Slide

103. Observable Subscriber
     Observable Subscriber
     subscriber.setProducer
     subscriber.setProducer
     

     View Slide

104. Observable Subscriber
     Observable Subscriber
     producer.request(n)
     producer.request(5)
     outstanding
     requested = 5
     

     View Slide

105. Observable Subscriber
     Observable Subscriber
     producer.request(n)
     producer.request(5)
     outstanding
     requested = 5
     

     View Slide

106. Observable Subscriber
     Observable Subscriber
     producer.request(n)
     producer.request(5)
     outstanding
     requested = 5
     

     View Slide

107. Observable Subscriber
     Observable Subscriber
     subscriber.onNext
     outstanding
     requested = 5
     

     View Slide

108. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 5
     subscriber.onNext
     

     View Slide

109. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 4
     subscriber.onNext
     

     View Slide

110. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 3
     subscriber.onNext
     

     View Slide

111. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 3
     subscriber.onNext
     producer.request(1)
     request(1)
     

     View Slide

112. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 4
     subscriber.onNext
     producer.request(1)
     

     View Slide

113. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 4
     subscriber.onNext
     subscriber.onNext
     

     View Slide

114. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 0
     subscriber.onNext
     

     View Slide

115. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 0
     subscriber.onNext
     

     View Slide

116. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 0
     subscriber.onNext
     

     View Slide

117. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 0
     subscriber.onNext
     

     View Slide

118. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 0
     subscriber.onNext
     

     View Slide

119. Observable Subscriber
     Observable Subscriber
     outstanding
     requested = 2
     subscriber.onNext
     request(2)
     producer.request(2)
     

     View Slide

120. someObservable.subscribe(new Subscriber() {
     @Override
     public void onStart() {
     request(1);
     }
     !
     @Override
     public void onCompleted() {
     // gracefully handle sequence-complete
     }
     !
     @Override
     public void onError(Throwable e) {
     // gracefully handle error
     }
     !
     @Override
     public void onNext(T n) {
     // do something with the emitted item "n"
     // request another item:
     request(1);
     }
     });
     

     View Slide

121. someObservable.subscribe(new Subscriber() {
     @Override
     public void onStart() {
     request(1);
     }
     !
     @Override
     public void onCompleted() {
     // gracefully handle sequence-complete
     }
     !
     @Override
     public void onError(Throwable e) {
     // gracefully handle error
     }
     !
     @Override
     public void onNext(T n) {
     // do something with the emitted item "n"
     // request another item:
     request(1);
     }
     });
     

     View Slide

122. someObservable.subscribe(new Subscriber() {
     @Override
     public void onStart() {
     request(1);
     }
     !
     @Override
     public void onCompleted() {
     // gracefully handle sequence-complete
     }
     !
     @Override
     public void onError(Throwable e) {
     // gracefully handle error
     }
     !
     @Override
     public void onNext(T n) {
     // do something with the emitted item "n"
     // request another item:
     request(1);
     }
     });
     

     View Slide

123. someObservable.subscribe(new Subscriber() {
     @Override
     public void onStart() {
     request(1024);
     }
     !
     @Override
     public void onCompleted() {
     // gracefully handle sequence-complete
     }
     !
     @Override
     public void onError(Throwable e) {
     // gracefully handle error
     }
     !
     @Override
     public void onNext(T n) {
     enqueue(n);
     // elsewhere the queue is drained
     // and request(m) is called
     }
     !
     });
     

     View Slide

124. someObservable.subscribe(new Subscriber() {
     @Override
     public void onStart() {
     request(1024);
     }
     !
     @Override
     public void onCompleted() {
     // gracefully handle sequence-complete
     }
     !
     @Override
     public void onError(Throwable e) {
     // gracefully handle error
     }
     !
     @Override
     public void onNext(T n) {
     enqueue(n);
     // elsewhere the queue is drained
     // and request(m) is called
     }
     !
     });
     

     View Slide

125. someObservable.subscribe(new Subscriber() {
     @Override
     public void onStart() {
     request(1024);
     }
     !
     @Override
     public void onCompleted() {
     // gracefully handle sequence-complete
     }
     !
     @Override
     public void onError(Throwable e) {
     // gracefully handle error
     }
     !
     @Override
     public void onNext(T n) {
     enqueue(n);
     // elsewhere the queue is drained
     // and request(m) is called
     }
     !
     });
     

     View Slide

126. Reactive Pull
     hot vs cold?
     

     View Slide

127. Reactive Pull
     … or when source doesn’t support request(n)?
     

     View Slide

128. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);
     

     View Slide

129. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);
     

     View Slide

130. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);
     

     View Slide

131. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);
     

     View Slide

132. hotSourceStream.onBackpressureDrop().observeOn(aScheduler);
     

     View Slide

133. stream.onBackpressure(strategy).subscribe
     

     View Slide

134. distributed
     

     View Slide

135. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

136. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

137. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

138. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

139. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

140. (movieId)
     movieId=12345 movieId=34567
     

     View Slide

141. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

142. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

143. Stage 1 Stage 2
     groupBy
     Event Streams
     sink
     

     View Slide

144. Stage 1 Stage 2
     12345
     56789
     34567
     

     View Slide

145. Stage 1 Stage 2
     12345
     56789
     34567
     

     View Slide

146. Stage 1 Stage 2
     12345
     56789
     34567
     

     View Slide

147. Stage 1 Stage 2
     12345
     56789
     34567
     

     View Slide

148. Stage 1 Stage 2
     12345
     56789
     34567
     12345
     56789
     34567
     

     View Slide

149. Stage 1 Stage 2
     12345
     56789
     34567
     12345
     56789
     34567
     

     View Slide

150. Stage 1 Stage 2
     12345
     56789
     34567
     12345
     56789
     34567
     

     View Slide

151. Stage 1 Stage 2
     12345
     56789
     34567
     12345
     56789
     34567
     

     View Slide

152. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

153. 10mins || 1000
     4:40-4:50pm
     4:50-5:00pm
     5:00-5:07pm
     (burst to 1000)
     5:07-5:17pm
     

     View Slide

154. 10mins || 1000
     4:40-4:50pm
     4:50-5:00pm
     5:00-5:07pm
     (burst to 1000)
     5:07-5:17pm
     

     View Slide

155. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

156. flatMap
     

     View Slide

157. flatMap
     

     View Slide

158. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

159. View Slide

160. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

161. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

162. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

163. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

164. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

165. MantisJob
     .source(NetflixSources.moviePlayAttempts())
     .stage(playAttempts -> {
     return playAttempts.groupBy(playAttempt -> {
     return playAttempt.getMovieId();
     })
     })
     .stage(playAttemptsByMovieId -> {
     playAttemptsByMovieId
     .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
     .flatMap(windowOfPlayAttempts -> {
     return windowOfPlayAttempts
     .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
     experiment.updateFailRatio(playAttempt);
     experiment.updateExamples(playAttempt);
     return experiment;
     }).doOnNext(experiment -> {
     logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
     }).filter(experiment -> {
     return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
     }).map(experiment -> {
     return new FailReport(experiment, runCorrelations(experiment.getExamples()));
     }).doOnNext(report -> {
     logToHistorical("Failure report", report.getId(), report); // log for offline analysis
     })
     })
     })
     .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
     

     View Slide

166. stream.onBackpressure(strategy?).subscribe
     

     View Slide

167. stream.onBackpressure(buffer).subscribe
     

     View Slide

168. stream.onBackpressure(drop).subscribe
     

     View Slide

169. stream.onBackpressure(sample).subscribe
     

     View Slide

170. stream.onBackpressure(scaleHorizontally).subscribe
     

     View Slide

171. finite
     infinite
     distributed
     

     View Slide

172. Single Multiple
     Sync T getData() Iterable getData()
     Async Future getData() Observable getData()
     

     View Slide

173. RxJava
     http://github.com/ReactiveX/RxJava
     http://reactivex.io
     

     View Slide

174. !
     Reactive Programming in the Netflix API with RxJava http://techblog.netflix.com/2013/02/rxjava-netflix-api.html
     !
     Optimizing the Netflix API http://techblog.netflix.com/2013/01/optimizing-netflix-api.html
     !
     Reactive Extensions (Rx) http://www.reactivex.io
     !
     Reactive Streams https://github.com/reactive-streams/reactive-streams
     !
     !
     !
     !
     !
     Ben Christensen
     @benjchristensen
     RxJava
     https://github.com/ReactiveX/RxJava
     @RxJava
     RxJS
     http://reactive-extensions.github.io/RxJS/
     @ReactiveX
     jobs.netflix.com
     

     View Slide