Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Reactive Streams with Rx at JavaOne 2014

Reactive Streams with Rx at JavaOne 2014

Learn how Netflix applies the Rx programming model to streams of data in finite, infinite and distributed infinite stream uses cases including a distributed system doing anomaly detection.

The presentation digs into the details of flow control and various approaches to achieve it including "reactive pull", a dynamic push/pull approach that composes backpressure into the streams.

Presented at JavaOne 2014: https://oracleus.activeevents.com/2014/connect/sessionDetail.ww?SESSION_ID=5749

Video: http://www.parleys.com/play/543f8d5be4b06e1184ae4106

More information can be found at http://reactivex.io

Ben Christensen

September 29, 2014
Tweet

More Decks by Ben Christensen

Other Decks in Programming

Transcript

  1. Ben Christensen Developer – Edge Engineering at Netflix @benjchristensen !

    ! ! ! ! ! http://techblog.netflix.com/ JavaOne - September 2014 Reactive Streams with Rx
  2. public interface Observer<T> { public abstract void onCompleted() public abstract

    void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Observer<? super T> subscriber) } Lazy Cold or Hot Asynchronous Push *onNext (onError | onCompleted)?
  3. public interface Observer<T> { public abstract void onCompleted() public abstract

    void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Observer<? super T> subscriber) } Lazy Cold or Hot Asynchronous (or Synchronous) Push (or Pull) *onNext (onError | onCompleted)?
  4. public interface Observer<T> { public abstract void onCompleted() public abstract

    void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Observer<? super T> subscriber) } Lazy Cold or Hot Asynchronous (or Synchronous) Push (or Pull) *onNext (onError | onCompleted)?
  5. public interface Subscriber<T> implements Observer<T> { public abstract void onCompleted()

    public abstract void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) } Lazy Cold or Hot Asynchronous (or Synchronous) Push (or Pull) *onNext (onError | onCompleted)?
  6. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  7. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  8. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  9. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  10. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } “reactive pull” dynamic push/pull for flow control (backpressure)
  11. public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f)

    public Subscription subscribe(Subscriber<? super T> subscriber) } public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  12. public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f)

    public Subscription subscribe(Subscriber<? super T> subscriber) } public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  13. public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f)

    public Subscription subscribe(Subscriber<? super T> subscriber) } public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  14. Observable.create(subscriber -> { subscriber.onNext("Hello world!"); subscriber.onCompleted(); }).forEach(System.out::println); synchronous single value

    public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  15. Observable.create(subscriber -> { subscriber.onNext("Hello"); subscriber.onNext("world"); subscriber.onNext("!"); subscriber.onCompleted(); }).forEach(System.out::println); synchronous multiple

    values public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  16. Observable.create(subscriber -> { try { subscriber.onNext(doSomething()); subscriber.onCompleted(); } catch (Throwable

    e) { subscriber.onError(e); } }).subscribeOn(Schedulers.io()) .forEach(System.out::println); asynchronous single value error notification public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  17. Observable.create(subscriber -> { int i = 0; while (!subscriber.isUnsubscribed()) {

    subscriber.onNext(i++); } }).take(10).forEach(System.out::println); synchronous multiple values with unsubscribe public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  18. Observable.create(subscriber -> { AtomicInteger i = new AtomicInteger(); AtomicLong requested

    = new AtomicLong(); subscriber.setProducer(r -> { if (requested.getAndAdd(r) == 0) { do { if (subscriber.isUnsubscribed()) { break; } subscriber.onNext(i.incrementAndGet()); } while (requested.decrementAndGet() > 0); } }); }).observeOn(Schedulers.newThread()) .take(10).forEach(System.out::println); synchronous multiple values flow control with “reactive pull” backpressure thread scheduling public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  19. Observable.create(subscriber -> { AtomicInteger i = new AtomicInteger(); AtomicLong requested

    = new AtomicLong(); subscriber.setProducer(r -> { if (requested.getAndAdd(r) == 0) { do { if (subscriber.isUnsubscribed()) { break; } subscriber.onNext(i.incrementAndGet()); } while (requested.decrementAndGet() > 0); } }); }).observeOn(Schedulers.newThread()) .take(10).forEach(System.out::println); public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  20. Observable.from(iterable) .observeOn(Schedulers.newThread()) .take(10).forEach(System.out::println); public static interface OnSubscribe<T> { public void

    call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  21. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  22. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  23. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  24.  Observable<R>  b  =  Observable<T>.flatMap({  T  t  -­‐>      

         Observable<R>  r  =  ...  transform  t  ...          return  r;    }) flatMap
  25.  Observable<R>  b  =  Observable<T>.flatMap({  T  t  -­‐>      

         Observable<R>  r  =  ...  transform  t  ...          return  r;    }) flatMap
  26. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  27. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  28. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  29. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  30. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  31. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  32. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  33. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  34. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  35.        Observable.zip(a,  b,  {  a,  b,  -­‐>  

                 ...  operate  on  values  from  both  a  &  b  ...              return  [a,  b];  //  i.e.  return  tuple          })
  36. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  37. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  38. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  39. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  40. class  VideoService  {        def  Observable<VideoList>  getPersonalizedListOfMovies(userId);  

         def  Observable<VideoBookmark>  getBookmark(userId,  videoId);        def  Observable<VideoRating>  getRating(userId,  videoId);        def  Observable<VideoMetadata>  getMetadata(videoId);   } an observable api: finite Treat everything like a stream
  41. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); } 1 2 3 4 5 6
  42. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); } 3 4 5
  43. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  44. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  45. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  46. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  47. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  48. data => {"rollingCountFallbackFailure":0,"rollingCountFallbackSuccess": 0,"propertyValue_circuitBreakerRequestVolumeThreshold":"20","propertyValue_circuitBreakerForceOpen":false,"propertyValue_metricsRol lingStatisticalWindowInMilliseconds":"10000","latencyTotal_mean":32,"type":"HystrixCommand","rollingCountResponsesFromCache": 0,"rollingCountTimeout":0,"propertyValue_executionIsolationStrategy":"SEMAPHORE","instanceId":"34567","rollingCountFailure": 0,"rollingCountExceptionsThrown":0,"latencyExecute_mean":32,"isCircuitBreakerOpen":false,"errorCount": 0,"rollingCountSemaphoreRejected":0,"latencyTotal":{"0":0,"25":0,"50":12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100": 1560},"requestCount":1376,"rollingCountCollapsedRequests":94516,"rollingCountShortCircuited":0,"latencyExecute":{"0":0,"25":0,"50": 12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100":

    1560},"propertyValue_circuitBreakerSleepWindowInMilliseconds":"5000","currentConcurrentExecutionCount": 0,"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests":"10","errorPercentage":0,"rollingCountThreadPoolRejected": 0,"propertyValue_circuitBreakerEnabled":true,"propertyValue_executionIsolationThreadInterruptOnTimeout":true,"propertyValue_request CacheEnabled":true,"rollingCountFallbackRejection":0,"propertyValue_requestLogEnabled":true,"rollingCountSuccess": 1432,"propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests":"10","propertyValue_circuitBreakerErrorThresholdPercentage":"5 0","propertyValue_circuitBreakerForceClosed":false,"name":"GetPredictions","reportingHosts": 4,"propertyValue_executionIsolationThreadPoolKeyOverride":"null","propertyValue_executionIsolationThreadTimeoutInMilliseconds":"100 0"} data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1} data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}
  49. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  50. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  51. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  52. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  53. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  54. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] [] https://gist.github.com/benjchristensen/e4524a308456f3c21c0b
  55. Push when consumer keeps up with producer. ! Switch to

    Pull when consumer is slow. ! Bound all* queues.
  56. Push when consumer keeps up with producer. ! Switch to

    Pull when consumer is slow. ! Bound all* queues. *vertically, not horizontally
  57. Observable Subscriber Observable.create(subscriber -> { // emit to subscriber }).subscribe(new

    Subscriber<String>() { ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! });
  58. observable.subscribe Observable.create(subscriber -> { // emit to subscriber }).subscribe(new Subscriber<String>()

    { ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! });
  59. observable.subscribe subscriber.setProducer Observable.create(subscriber -> { subscriber.setProducer(request -> { }); }).subscribe(new

    Subscriber<String>() { ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! });
  60. Observable.create(subscriber -> { subscriber.setProducer(request -> { }); }).subscribe(new Subscriber<String>() {

    ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! }); observable.subscribe subscriber.setProducer producer.request(n)
  61. Observable.create(subscriber -> { subscriber.setProducer(request -> { subscriber.onNext(t); // 5 times

    }); }).subscribe(new Subscriber<String>() { ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! }); observable.subscribe subscriber.setProducer producer.request(n) subscriber.onNext(t) …
  62. Observable.create(subscriber -> { subscriber.setProducer(request -> { subscriber.onNext(t); // 5 times

    }); }).subscribe(new Subscriber<String>() { ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { doStuffWith(t); if(requested—- == 0) request(5); } ! }); observable.subscribe subscriber.setProducer producer.request(n) subscriber.onNext(t) … producer.request(n)
  63. Observable.create(subscriber -> { subscriber.setProducer(request -> { … previous logic …

    subscriber.onCompleted(); }); }).subscribe(new Subscriber<String>() { ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { doStuffWith(t); if(requested—- == 0) request(5); } ! }); observable.subscribe subscriber.setProducer producer.request(n) subscriber.onNext(t) … producer.request(n) subscriber.onCompleted/onError unsubscribe
  64. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { // do something with the emitted item "n" // request another item: request(1); } });
  65. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { // do something with the emitted item "n" // request another item: request(1); } });
  66. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { // do something with the emitted item "n" // request another item: request(1); } });
  67. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1024); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { enqueue(n); // elsewhere the queue is drained // and request(m) is called } ! });
  68. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1024); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { enqueue(n); // elsewhere the queue is drained // and request(m) is called } ! });
  69. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1024); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { enqueue(n); // elsewhere the queue is drained // and request(m) is called } ! });
  70. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  71. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  72. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  73. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  74. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  75. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  76. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  77. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  78. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  79. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  80. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  81. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  82. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  83. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  84. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  85. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  86. ! Reactive Programming in the Netflix API with RxJava http://techblog.netflix.com/2013/02/rxjava-netflix-api.html

    ! Optimizing the Netflix API http://techblog.netflix.com/2013/01/optimizing-netflix-api.html ! Reactive Extensions (Rx) http://www.reactivex.io ! Reactive Streams https://github.com/reactive-streams/reactive-streams ! ! ! ! ! Ben Christensen @benjchristensen RxJava https://github.com/ReactiveX/RxJava @RxJava RxJS http://reactive-extensions.github.io/RxJS/ @ReactiveX jobs.netflix.com