$30 off During Our Annual Pro Sale. View Details »

Reactive Streams with Rx at JavaOne 2014

Reactive Streams with Rx at JavaOne 2014

Learn how Netflix applies the Rx programming model to streams of data in finite, infinite and distributed infinite stream uses cases including a distributed system doing anomaly detection.

The presentation digs into the details of flow control and various approaches to achieve it including "reactive pull", a dynamic push/pull approach that composes backpressure into the streams.

Presented at JavaOne 2014: https://oracleus.activeevents.com/2014/connect/sessionDetail.ww?SESSION_ID=5749

Video: http://www.parleys.com/play/543f8d5be4b06e1184ae4106

More information can be found at http://reactivex.io

Ben Christensen

September 29, 2014
Tweet

More Decks by Ben Christensen

Other Decks in Programming

Transcript

  1. Ben Christensen
    Developer – Edge Engineering at Netflix
    @benjchristensen
    !
    !
    !
    !
    !
    !
    http://techblog.netflix.com/
    JavaOne - September 2014
    Reactive Streams with Rx

    View Slide

  2. View Slide

  3. RxJava
    http://github.com/ReactiveX/RxJava
    http://reactivex.io

    View Slide

  4. View Slide

  5. Single Multiple
    Sync T getData() Iterable getData()
    Async Future getData() Observable getData()

    View Slide

  6. public interface Observer {
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Observer super T> subscriber)
    }
    Lazy
    Cold or Hot
    Asynchronous
    Push
    *onNext
    (onError | onCompleted)?

    View Slide

  7. public interface Observer {
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Observer super T> subscriber)
    }
    Lazy
    Cold or Hot
    Asynchronous (or Synchronous)
    Push (or Pull)
    *onNext
    (onError | onCompleted)?

    View Slide

  8. public interface Observer {
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Observer super T> subscriber)
    }
    Lazy
    Cold or Hot
    Asynchronous (or Synchronous)
    Push (or Pull)
    *onNext
    (onError | onCompleted)?

    View Slide

  9. public interface Subscriber implements Observer {
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    Lazy
    Cold or Hot
    Asynchronous (or Synchronous)
    Push (or Pull)
    *onNext
    (onError | onCompleted)?

    View Slide

  10. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  11. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  12. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  13. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  14. public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    “reactive pull”
    dynamic push/pull
    for flow control (backpressure)

    View Slide

  15. public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  16. public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  17. public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }

    View Slide

  18. Observable.create(subscriber -> {
    subscriber.onNext("Hello world!");
    subscriber.onCompleted();
    }).forEach(System.out::println);
    synchronous
    single value
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  19. Observable.create(subscriber -> {
    subscriber.onNext("Hello");
    subscriber.onNext("world");
    subscriber.onNext("!");
    subscriber.onCompleted();
    }).forEach(System.out::println);
    synchronous
    multiple values
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  20. Observable.create(subscriber -> {
    try {
    subscriber.onNext(doSomething());
    subscriber.onCompleted();
    } catch (Throwable e) {
    subscriber.onError(e);
    }
    }).subscribeOn(Schedulers.io())
    .forEach(System.out::println);
    asynchronous
    single value
    error notification
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  21. Observable.create(subscriber -> {
    int i = 0;
    while (!subscriber.isUnsubscribed()) {
    subscriber.onNext(i++);
    }
    }).take(10).forEach(System.out::println);
    synchronous
    multiple values
    with unsubscribe
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  22. Observable.create(subscriber -> {
    AtomicInteger i = new AtomicInteger();
    AtomicLong requested = new AtomicLong();
    subscriber.setProducer(r -> {
    if (requested.getAndAdd(r) == 0) {
    do {
    if (subscriber.isUnsubscribed()) { break; }
    subscriber.onNext(i.incrementAndGet());
    } while (requested.decrementAndGet() > 0);
    }
    });
    }).observeOn(Schedulers.newThread())
    .take(10).forEach(System.out::println);
    synchronous
    multiple values
    flow control with
    “reactive pull”
    backpressure
    thread scheduling
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  23. Observable.create(subscriber -> {
    AtomicInteger i = new AtomicInteger();
    AtomicLong requested = new AtomicLong();
    subscriber.setProducer(r -> {
    if (requested.getAndAdd(r) == 0) {
    do {
    if (subscriber.isUnsubscribed()) { break; }
    subscriber.onNext(i.incrementAndGet());
    } while (requested.decrementAndGet() > 0);
    }
    });
    }).observeOn(Schedulers.newThread())
    .take(10).forEach(System.out::println);
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  24. Observable.from(iterable)
    .observeOn(Schedulers.newThread())
    .take(10).forEach(System.out::println);
    public static interface OnSubscribe {
    public void call(Subscriber super T> subscriber);
    }
    public abstract class Subscriber implements Observer, Subscription
    public abstract void onCompleted()
    public abstract void onError(Throwable e)
    public abstract void onNext(T t)
    !
    public final void add(Subscription s)
    public void setProducer(Producer producer)
    }
    !
    public interface Producer {
    public void request(long n);
    }
    public class Observable {
    public static Observable create(OnSubscribe f)
    public Subscription subscribe(Subscriber super T> subscriber)
    }

    View Slide

  25. finite

    View Slide

  26. infinite
    finite

    View Slide

  27. finite
    infinite
    distributed

    View Slide

  28. View Slide

  29. View Slide

  30. View Slide

  31. View Slide

  32. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  33. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  34. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  35.  Observable  b  =  Observable.flatMap({  T  t  -­‐>    
           Observable  r  =  ...  transform  t  ...  
           return  r;  
     })
    flatMap

    View Slide

  36.  Observable  b  =  Observable.flatMap({  T  t  -­‐>    
           Observable  r  =  ...  transform  t  ...  
           return  r;  
     })
    flatMap

    View Slide

  37. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  38. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  39. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  40. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  41. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  42. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  43. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  44. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  45. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  46.        Observable.zip(a,  b,  {  a,  b,  -­‐>    
               ...  operate  on  values  from  both  a  &  b  ...  
               return  [a,  b];  //  i.e.  return  tuple  
           })

    View Slide

  47. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  48. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  49. View Slide

  50. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  51. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  52. class  VideoService  {  
         def  Observable  getPersonalizedListOfMovies(userId);  
         def  Observable  getBookmark(userId,  videoId);  
         def  Observable  getRating(userId,  videoId);  
         def  Observable  getMetadata(videoId);  
    }
    an observable api:
    finite
    Treat everything like a stream

    View Slide

  53. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    1
    2
    3
    4
    5
    6

    View Slide

  54. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }
    3
    4
    5

    View Slide

  55. public Observable handle(HttpServerRequest request, HttpServerResponse response) {
    // first request User object
    return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> {
    // then fetch personal catalog
    Observable> catalog = new PersonalizedCatalogCommand(user).observe()
    .flatMap(catalogList -> {
    return catalogList.videos().> flatMap(video -> {
    Observable bookmark = new BookmarkCommand(video).observe();
    Observable rating = new RatingsCommand(video).observe();
    Observable metadata = new VideoMetadataCommand(video).observe();
    return Observable.zip(bookmark, rating, metadata, (b, r, m) -> {
    return combineVideoData(video, b, r, m);
    });
    });
    });
    // and fetch social data in parallel
    Observable> social = new SocialCommand(user).observe().map(s -> {
    return s.getDataAsMap();
    });
    // merge the results
    return Observable.merge(catalog, social);
    }).flatMap(data -> {
    // output as SSE as we get back the data (no waiting until all is done)
    return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data)));
    });
    }

    View Slide

  56. infinite

    View Slide

  57. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }

    View Slide

  58. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }

    View Slide

  59. View Slide

  60. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }

    View Slide

  61. View Slide

  62. private static Observable>>
    aggregateUsingPivot(Observable>> instanceStreams) {
    return instanceStreams.map(instanceStream -> {
    return createGroupedObservable(instanceStream.getKey(), instanceStream
    .groupBy((Map json) -> {
    return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name")));
    }));
    }).lift(OperatorPivot.create()).map(commandGroup -> {
    // merge all instances per group into a single stream of deltas and sum them
    return createGroupedObservable(commandGroup.getKey(),
    commandGroup.flatMap(instanceGroup -> {
    return instanceGroup.startWith(Collections. emptyMap())
    .buffer(2, 1)
    .map(StreamAggregator::previousAndCurrentToDelta)
    .filter(data -> data != null && !data.isEmpty());
    }).scan(new HashMap(), StreamAggregator::sumOfDelta)
    .skip(1));
    });
    }

    View Slide

  63. View Slide

  64. data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}

    View Slide

  65. data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}

    View Slide

  66. data => {"rollingCountFallbackFailure":0,"rollingCountFallbackSuccess":
    0,"propertyValue_circuitBreakerRequestVolumeThreshold":"20","propertyValue_circuitBreakerForceOpen":false,"propertyValue_metricsRol
    lingStatisticalWindowInMilliseconds":"10000","latencyTotal_mean":32,"type":"HystrixCommand","rollingCountResponsesFromCache":
    0,"rollingCountTimeout":0,"propertyValue_executionIsolationStrategy":"SEMAPHORE","instanceId":"34567","rollingCountFailure":
    0,"rollingCountExceptionsThrown":0,"latencyExecute_mean":32,"isCircuitBreakerOpen":false,"errorCount":
    0,"rollingCountSemaphoreRejected":0,"latencyTotal":{"0":0,"25":0,"50":12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100":
    1560},"requestCount":1376,"rollingCountCollapsedRequests":94516,"rollingCountShortCircuited":0,"latencyExecute":{"0":0,"25":0,"50":
    12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100":
    1560},"propertyValue_circuitBreakerSleepWindowInMilliseconds":"5000","currentConcurrentExecutionCount":
    0,"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests":"10","errorPercentage":0,"rollingCountThreadPoolRejected":
    0,"propertyValue_circuitBreakerEnabled":true,"propertyValue_executionIsolationThreadInterruptOnTimeout":true,"propertyValue_request
    CacheEnabled":true,"rollingCountFallbackRejection":0,"propertyValue_requestLogEnabled":true,"rollingCountSuccess":
    1432,"propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests":"10","propertyValue_circuitBreakerErrorThresholdPercentage":"5
    0","propertyValue_circuitBreakerForceClosed":false,"name":"GetPredictions","reportingHosts":
    4,"propertyValue_executionIsolationThreadPoolKeyOverride":"null","propertyValue_executionIsolationThreadTimeoutInMilliseconds":"100
    0"}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1}
    data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}

    View Slide

  67. Flow Control

    View Slide

  68. Flow Control
    (backpressure)

    View Slide

  69. no backpressure needed
    Observable.from(iterable).take(1000).map(i -> "value_" + i).subscribe(System.out::println);

    View Slide

  70. Observable.from(iterable).take(1000).map(i -> "value_" + i).subscribe(System.out::println);
    no backpressure needed
    synchronous on same thread
    (no queueing)

    View Slide

  71. Observable.from(iterable).take(1000).map(i -> "value_" + i)
    .observeOn(Schedulers.computation()).subscribe(System.out::println);
    backpressure needed

    View Slide

  72. Observable.from(iterable).take(1000).map(i -> "value_" + i)
    .observeOn(Schedulers.computation()).subscribe(System.out::println);
    backpressure needed
    asynchronous
    (queueing)

    View Slide

  73. Flow Control Options

    View Slide

  74. Block
    (callstack blocking and/or park the thread)

    View Slide

  75. Temporal Operators
    (batch or drop data using time)

    View Slide

  76. Observable.range(1, 1000000).sample(10, TimeUnit.MILLISECONDS).forEach(System.out::println);
    110584
    242165
    544453
    942880

    View Slide

  77. Observable.range(1, 1000000).throttleFirst(10, TimeUnit.MILLISECONDS).forEach(System.out::println);
    1
    55463
    163962
    308545
    457445
    592638
    751789
    897159

    View Slide

  78. Observable.range(1, 1000000).debounce(10, TimeUnit.MILLISECONDS).forEach(System.out::println);
    1000000

    View Slide

  79. Observable.range(1, 1000000).buffer(10, TimeUnit.MILLISECONDS)
    .toBlocking().forEach(list -> System.out.println("batch: " + list.size()));
    batch: 71141
    batch: 49488
    batch: 141147
    batch: 141432
    batch: 195920
    batch: 240462
    batch: 160410

    View Slide

  80. View Slide

  81. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []

    View Slide

  82. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []

    View Slide

  83. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []

    View Slide

  84. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []

    View Slide

  85. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []

    View Slide

  86. /* The following will emit a buffered list as it is debounced */
    // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe
    Observable burstStream = intermittentBursts().take(20).publish().refCount();
    // then we get the debounced version
    Observable debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS);
    // then the buffered one that uses the debounced stream to demark window start/stop
    Observable> buffered = burstStream.buffer(debounced);
    // then we subscribe to the buffered stream so it does what we want
    buffered.toBlocking().forEach(System.out::println);
    [0, 1, 2]
    [0, 1, 2]
    [0, 1, 2, 3, 4, 5, 6]
    [0, 1, 2, 3, 4]
    [0, 1]
    []
    https://gist.github.com/benjchristensen/e4524a308456f3c21c0b

    View Slide

  87. Observable.range(1, 1000000).window(50, TimeUnit.MILLISECONDS)
    .flatMap(window -> window.count())
    .toBlocking().forEach(count -> System.out.println("num items: " + count));
    num items: 477769
    num items: 155463
    num items: 366768

    View Slide

  88. Observable.range(1, 1000000).window(500000)
    .flatMap(window -> window.count())
    .toBlocking().forEach(count -> System.out.println("num items: " + count));
    num items: 500000
    num items: 500000

    View Slide

  89. Reactive Pull
    (dynamic push-pull)

    View Slide

  90. Push when consumer
    keeps up with producer.
    !
    Switch to Pull
    when consumer is slow.
    !
    Bound all* queues.

    View Slide

  91. Push when consumer
    keeps up with producer.
    !
    Switch to Pull
    when consumer is slow.
    !
    Bound all* queues.
    *vertically, not horizontally

    View Slide

  92. Observable Subscriber
    Observable.create(subscriber -> {
    // emit to subscriber
    }).subscribe(new Subscriber() {
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });

    View Slide

  93. observable.subscribe
    Observable.create(subscriber -> {
    // emit to subscriber
    }).subscribe(new Subscriber() {
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });

    View Slide

  94. observable.subscribe
    subscriber.setProducer
    Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    });
    }).subscribe(new Subscriber() {
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });

    View Slide

  95. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)

    View Slide

  96. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    subscriber.onNext(t); // 5 times
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    !
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    subscriber.onNext(t) …

    View Slide

  97. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    subscriber.onNext(t); // 5 times
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    doStuffWith(t);
    if(requested—- == 0) request(5);
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    subscriber.onNext(t) …
    producer.request(n)

    View Slide

  98. Observable.create(subscriber -> {
    subscriber.setProducer(request -> {
    … previous logic …
    subscriber.onCompleted();
    });
    }).subscribe(new Subscriber() {
    !
    public void onStart() {
    request(5);
    }
    !
    public void onCompleted() {
    // do stuff
    }
    !
    public void onError(Throwable e) {
    // handle error
    }
    !
    public void onNext(String t) {
    doStuffWith(t);
    if(requested—- == 0) request(5);
    }
    !
    });
    observable.subscribe
    subscriber.setProducer
    producer.request(n)
    subscriber.onNext(t) …
    producer.request(n)
    subscriber.onCompleted/onError
    unsubscribe

    View Slide

  99. Observable Subscriber
    Observable Subscriber
    observable.subscribe
    observable.subscribe

    View Slide

  100. Observable.from(iterable) .observeOn(Schedulers.computation())
    .subscribe(System.out::println);

    View Slide

  101. Observable.from(iterable) .observeOn(Schedulers.computation())
    .subscribe(System.out::println);

    View Slide

  102. Observable Subscriber
    Observable Subscriber
    observable.subscribe
    observable.subscribe

    View Slide

  103. Observable Subscriber
    Observable Subscriber
    subscriber.setProducer
    subscriber.setProducer

    View Slide

  104. Observable Subscriber
    Observable Subscriber
    producer.request(n)
    producer.request(5)
    outstanding
    requested = 5

    View Slide

  105. Observable Subscriber
    Observable Subscriber
    producer.request(n)
    producer.request(5)
    outstanding
    requested = 5

    View Slide

  106. Observable Subscriber
    Observable Subscriber
    producer.request(n)
    producer.request(5)
    outstanding
    requested = 5

    View Slide

  107. Observable Subscriber
    Observable Subscriber
    subscriber.onNext
    outstanding
    requested = 5

    View Slide

  108. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 5
    subscriber.onNext

    View Slide

  109. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 4
    subscriber.onNext

    View Slide

  110. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 3
    subscriber.onNext

    View Slide

  111. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 3
    subscriber.onNext
    producer.request(1)
    request(1)

    View Slide

  112. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 4
    subscriber.onNext
    producer.request(1)

    View Slide

  113. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 4
    subscriber.onNext
    subscriber.onNext

    View Slide

  114. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 0
    subscriber.onNext

    View Slide

  115. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 0
    subscriber.onNext

    View Slide

  116. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 0
    subscriber.onNext

    View Slide

  117. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 0
    subscriber.onNext

    View Slide

  118. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 0
    subscriber.onNext

    View Slide

  119. Observable Subscriber
    Observable Subscriber
    outstanding
    requested = 2
    subscriber.onNext
    request(2)
    producer.request(2)

    View Slide

  120. someObservable.subscribe(new Subscriber() {
    @Override
    public void onStart() {
    request(1);
    }
    !
    @Override
    public void onCompleted() {
    // gracefully handle sequence-complete
    }
    !
    @Override
    public void onError(Throwable e) {
    // gracefully handle error
    }
    !
    @Override
    public void onNext(T n) {
    // do something with the emitted item "n"
    // request another item:
    request(1);
    }
    });

    View Slide

  121. someObservable.subscribe(new Subscriber() {
    @Override
    public void onStart() {
    request(1);
    }
    !
    @Override
    public void onCompleted() {
    // gracefully handle sequence-complete
    }
    !
    @Override
    public void onError(Throwable e) {
    // gracefully handle error
    }
    !
    @Override
    public void onNext(T n) {
    // do something with the emitted item "n"
    // request another item:
    request(1);
    }
    });

    View Slide

  122. someObservable.subscribe(new Subscriber() {
    @Override
    public void onStart() {
    request(1);
    }
    !
    @Override
    public void onCompleted() {
    // gracefully handle sequence-complete
    }
    !
    @Override
    public void onError(Throwable e) {
    // gracefully handle error
    }
    !
    @Override
    public void onNext(T n) {
    // do something with the emitted item "n"
    // request another item:
    request(1);
    }
    });

    View Slide

  123. someObservable.subscribe(new Subscriber() {
    @Override
    public void onStart() {
    request(1024);
    }
    !
    @Override
    public void onCompleted() {
    // gracefully handle sequence-complete
    }
    !
    @Override
    public void onError(Throwable e) {
    // gracefully handle error
    }
    !
    @Override
    public void onNext(T n) {
    enqueue(n);
    // elsewhere the queue is drained
    // and request(m) is called
    }
    !
    });

    View Slide

  124. someObservable.subscribe(new Subscriber() {
    @Override
    public void onStart() {
    request(1024);
    }
    !
    @Override
    public void onCompleted() {
    // gracefully handle sequence-complete
    }
    !
    @Override
    public void onError(Throwable e) {
    // gracefully handle error
    }
    !
    @Override
    public void onNext(T n) {
    enqueue(n);
    // elsewhere the queue is drained
    // and request(m) is called
    }
    !
    });

    View Slide

  125. someObservable.subscribe(new Subscriber() {
    @Override
    public void onStart() {
    request(1024);
    }
    !
    @Override
    public void onCompleted() {
    // gracefully handle sequence-complete
    }
    !
    @Override
    public void onError(Throwable e) {
    // gracefully handle error
    }
    !
    @Override
    public void onNext(T n) {
    enqueue(n);
    // elsewhere the queue is drained
    // and request(m) is called
    }
    !
    });

    View Slide

  126. Reactive Pull
    hot vs cold?

    View Slide

  127. Reactive Pull
    … or when source doesn’t support request(n)?

    View Slide

  128. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);

    View Slide

  129. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);

    View Slide

  130. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);

    View Slide

  131. hotSourceStream.onBackpressureBuffer().observeOn(aScheduler);

    View Slide

  132. hotSourceStream.onBackpressureDrop().observeOn(aScheduler);

    View Slide

  133. stream.onBackpressure(strategy).subscribe

    View Slide

  134. distributed

    View Slide

  135. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  136. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  137. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  138. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  139. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  140. (movieId)
    movieId=12345 movieId=34567

    View Slide

  141. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  142. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  143. Stage 1 Stage 2
    groupBy
    Event Streams
    sink

    View Slide

  144. Stage 1 Stage 2
    12345
    56789
    34567

    View Slide

  145. Stage 1 Stage 2
    12345
    56789
    34567

    View Slide

  146. Stage 1 Stage 2
    12345
    56789
    34567

    View Slide

  147. Stage 1 Stage 2
    12345
    56789
    34567

    View Slide

  148. Stage 1 Stage 2
    12345
    56789
    34567
    12345
    56789
    34567

    View Slide

  149. Stage 1 Stage 2
    12345
    56789
    34567
    12345
    56789
    34567

    View Slide

  150. Stage 1 Stage 2
    12345
    56789
    34567
    12345
    56789
    34567

    View Slide

  151. Stage 1 Stage 2
    12345
    56789
    34567
    12345
    56789
    34567

    View Slide

  152. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  153. 10mins || 1000
    4:40-4:50pm
    4:50-5:00pm
    5:00-5:07pm
    (burst to 1000)
    5:07-5:17pm

    View Slide

  154. 10mins || 1000
    4:40-4:50pm
    4:50-5:00pm
    5:00-5:07pm
    (burst to 1000)
    5:07-5:17pm

    View Slide

  155. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  156. flatMap

    View Slide

  157. flatMap

    View Slide

  158. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  159. View Slide

  160. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  161. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  162. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  163. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  164. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  165. MantisJob
    .source(NetflixSources.moviePlayAttempts())
    .stage(playAttempts -> {
    return playAttempts.groupBy(playAttempt -> {
    return playAttempt.getMovieId();
    })
    })
    .stage(playAttemptsByMovieId -> {
    playAttemptsByMovieId
    .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts
    .flatMap(windowOfPlayAttempts -> {
    return windowOfPlayAttempts
    .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> {
    experiment.updateFailRatio(playAttempt);
    experiment.updateExamples(playAttempt);
    return experiment;
    }).doOnNext(experiment -> {
    logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis
    }).filter(experiment -> {
    return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get();
    }).map(experiment -> {
    return new FailReport(experiment, runCorrelations(experiment.getExamples()));
    }).doOnNext(report -> {
    logToHistorical("Failure report", report.getId(), report); // log for offline analysis
    })
    })
    })
    .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)

    View Slide

  166. stream.onBackpressure(strategy?).subscribe

    View Slide

  167. stream.onBackpressure(buffer).subscribe

    View Slide

  168. stream.onBackpressure(drop).subscribe

    View Slide

  169. stream.onBackpressure(sample).subscribe

    View Slide

  170. stream.onBackpressure(scaleHorizontally).subscribe

    View Slide

  171. finite
    infinite
    distributed

    View Slide

  172. Single Multiple
    Sync T getData() Iterable getData()
    Async Future getData() Observable getData()

    View Slide

  173. RxJava
    http://github.com/ReactiveX/RxJava
    http://reactivex.io

    View Slide

  174. !
    Reactive Programming in the Netflix API with RxJava http://techblog.netflix.com/2013/02/rxjava-netflix-api.html
    !
    Optimizing the Netflix API http://techblog.netflix.com/2013/01/optimizing-netflix-api.html
    !
    Reactive Extensions (Rx) http://www.reactivex.io
    !
    Reactive Streams https://github.com/reactive-streams/reactive-streams
    !
    !
    !
    !
    !
    Ben Christensen
    @benjchristensen
    RxJava
    https://github.com/ReactiveX/RxJava
    @RxJava
    RxJS
    http://reactive-extensions.github.io/RxJS/
    @ReactiveX
    jobs.netflix.com

    View Slide