Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Reactive Streams with Rx at JavaOne 2014

Sponsored · SiteGround - Reliable hosting with speed, security, and support you can count on. →

Reactive Streams with Rx at JavaOne 2014

Learn how Netflix applies the Rx programming model to streams of data in finite, infinite and distributed infinite stream uses cases including a distributed system doing anomaly detection.

The presentation digs into the details of flow control and various approaches to achieve it including "reactive pull", a dynamic push/pull approach that composes backpressure into the streams.

Presented at JavaOne 2014: https://oracleus.activeevents.com/2014/connect/sessionDetail.ww?SESSION_ID=5749

Video: http://www.parleys.com/play/543f8d5be4b06e1184ae4106

More information can be found at http://reactivex.io

Avatar for Ben Christensen

Ben Christensen

September 29, 2014

More Decks by Ben Christensen

Other Decks in Programming

Transcript

  1. Ben Christensen Developer – Edge Engineering at Netflix @benjchristensen !

    ! ! ! ! ! http://techblog.netflix.com/ JavaOne - September 2014 Reactive Streams with Rx
  2. public interface Observer<T> { public abstract void onCompleted() public abstract

    void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Observer<? super T> subscriber) } Lazy Cold or Hot Asynchronous Push *onNext (onError | onCompleted)?
  3. public interface Observer<T> { public abstract void onCompleted() public abstract

    void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Observer<? super T> subscriber) } Lazy Cold or Hot Asynchronous (or Synchronous) Push (or Pull) *onNext (onError | onCompleted)?
  4. public interface Observer<T> { public abstract void onCompleted() public abstract

    void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Observer<? super T> subscriber) } Lazy Cold or Hot Asynchronous (or Synchronous) Push (or Pull) *onNext (onError | onCompleted)?
  5. public interface Subscriber<T> implements Observer<T> { public abstract void onCompleted()

    public abstract void onError(Throwable e) public abstract void onNext(T t) } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) } Lazy Cold or Hot Asynchronous (or Synchronous) Push (or Pull) *onNext (onError | onCompleted)?
  6. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  7. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  8. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  9. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  10. public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void

    onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } “reactive pull” dynamic push/pull for flow control (backpressure)
  11. public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f)

    public Subscription subscribe(Subscriber<? super T> subscriber) } public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  12. public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f)

    public Subscription subscribe(Subscriber<? super T> subscriber) } public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  13. public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f)

    public Subscription subscribe(Subscriber<? super T> subscriber) } public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); }
  14. Observable.create(subscriber -> { subscriber.onNext("Hello world!"); subscriber.onCompleted(); }).forEach(System.out::println); synchronous single value

    public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  15. Observable.create(subscriber -> { subscriber.onNext("Hello"); subscriber.onNext("world"); subscriber.onNext("!"); subscriber.onCompleted(); }).forEach(System.out::println); synchronous multiple

    values public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  16. Observable.create(subscriber -> { try { subscriber.onNext(doSomething()); subscriber.onCompleted(); } catch (Throwable

    e) { subscriber.onError(e); } }).subscribeOn(Schedulers.io()) .forEach(System.out::println); asynchronous single value error notification public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  17. Observable.create(subscriber -> { int i = 0; while (!subscriber.isUnsubscribed()) {

    subscriber.onNext(i++); } }).take(10).forEach(System.out::println); synchronous multiple values with unsubscribe public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  18. Observable.create(subscriber -> { AtomicInteger i = new AtomicInteger(); AtomicLong requested

    = new AtomicLong(); subscriber.setProducer(r -> { if (requested.getAndAdd(r) == 0) { do { if (subscriber.isUnsubscribed()) { break; } subscriber.onNext(i.incrementAndGet()); } while (requested.decrementAndGet() > 0); } }); }).observeOn(Schedulers.newThread()) .take(10).forEach(System.out::println); synchronous multiple values flow control with “reactive pull” backpressure thread scheduling public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  19. Observable.create(subscriber -> { AtomicInteger i = new AtomicInteger(); AtomicLong requested

    = new AtomicLong(); subscriber.setProducer(r -> { if (requested.getAndAdd(r) == 0) { do { if (subscriber.isUnsubscribed()) { break; } subscriber.onNext(i.incrementAndGet()); } while (requested.decrementAndGet() > 0); } }); }).observeOn(Schedulers.newThread()) .take(10).forEach(System.out::println); public static interface OnSubscribe<T> { public void call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  20. Observable.from(iterable) .observeOn(Schedulers.newThread()) .take(10).forEach(System.out::println); public static interface OnSubscribe<T> { public void

    call(Subscriber<? super T> subscriber); } public abstract class Subscriber<T> implements Observer<T>, Subscription public abstract void onCompleted() public abstract void onError(Throwable e) public abstract void onNext(T t) ! public final void add(Subscription s) public void setProducer(Producer producer) } ! public interface Producer { public void request(long n); } public class Observable<T> { public static <T> Observable<T> create(OnSubscribe<T> f) public Subscription subscribe(Subscriber<? super T> subscriber) }
  21. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  22. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  23. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  24.  Observable<R>  b  =  Observable<T>.flatMap({  T  t  -­‐>      

         Observable<R>  r  =  ...  transform  t  ...          return  r;    }) flatMap
  25.  Observable<R>  b  =  Observable<T>.flatMap({  T  t  -­‐>      

         Observable<R>  r  =  ...  transform  t  ...          return  r;    }) flatMap
  26. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  27. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  28. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  29. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  30. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  31. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  32. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  33. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  34. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  35.        Observable.zip(a,  b,  {  a,  b,  -­‐>  

                 ...  operate  on  values  from  both  a  &  b  ...              return  [a,  b];  //  i.e.  return  tuple          })
  36. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  37. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  38. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  39. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  40. class  VideoService  {        def  Observable<VideoList>  getPersonalizedListOfMovies(userId);  

         def  Observable<VideoBookmark>  getBookmark(userId,  videoId);        def  Observable<VideoRating>  getRating(userId,  videoId);        def  Observable<VideoMetadata>  getMetadata(videoId);   } an observable api: finite Treat everything like a stream
  41. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); } 1 2 3 4 5 6
  42. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); } 3 4 5
  43. public Observable<Void> handle(HttpServerRequest<ByteBuf> request, HttpServerResponse<ByteBuf> response) { // first request

    User object return new UserCommand(request.getQueryParameters().get("userId")).observe().flatMap(user -> { // then fetch personal catalog Observable<Map<String, Object>> catalog = new PersonalizedCatalogCommand(user).observe() .flatMap(catalogList -> { return catalogList.videos().<Map<String, Object>> flatMap(video -> { Observable<Bookmark> bookmark = new BookmarkCommand(video).observe(); Observable<Rating> rating = new RatingsCommand(video).observe(); Observable<VideoMetadata> metadata = new VideoMetadataCommand(video).observe(); return Observable.zip(bookmark, rating, metadata, (b, r, m) -> { return combineVideoData(video, b, r, m); }); }); }); // and fetch social data in parallel Observable<Map<String, Object>> social = new SocialCommand(user).observe().map(s -> { return s.getDataAsMap(); }); // merge the results return Observable.merge(catalog, social); }).flatMap(data -> { // output as SSE as we get back the data (no waiting until all is done) return response.writeAndFlush(new ServerSentEvent(SimpleJson.mapToJson(data))); }); }
  44. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  45. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  46. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  47. private static Observable<GroupedObservable<TypeAndNameKey, Map<String, Object>>> aggregateUsingPivot(Observable<GroupedObservable<InstanceKey, Map<String, Object>>> instanceStreams) {

    return instanceStreams.map(instanceStream -> { return createGroupedObservable(instanceStream.getKey(), instanceStream .groupBy((Map<String, Object> json) -> { return TypeAndNameKey.from(String.valueOf(json.get("type")), String.valueOf(json.get("name"))); })); }).lift(OperatorPivot.create()).map(commandGroup -> { // merge all instances per group into a single stream of deltas and sum them return createGroupedObservable(commandGroup.getKey(), commandGroup.flatMap(instanceGroup -> { return instanceGroup.startWith(Collections.<String, Object> emptyMap()) .buffer(2, 1) .map(StreamAggregator::previousAndCurrentToDelta) .filter(data -> data != null && !data.isEmpty()); }).scan(new HashMap<String, Object>(), StreamAggregator::sumOfDelta) .skip(1)); }); }
  48. data => {"rollingCountFallbackFailure":0,"rollingCountFallbackSuccess": 0,"propertyValue_circuitBreakerRequestVolumeThreshold":"20","propertyValue_circuitBreakerForceOpen":false,"propertyValue_metricsRol lingStatisticalWindowInMilliseconds":"10000","latencyTotal_mean":32,"type":"HystrixCommand","rollingCountResponsesFromCache": 0,"rollingCountTimeout":0,"propertyValue_executionIsolationStrategy":"SEMAPHORE","instanceId":"34567","rollingCountFailure": 0,"rollingCountExceptionsThrown":0,"latencyExecute_mean":32,"isCircuitBreakerOpen":false,"errorCount": 0,"rollingCountSemaphoreRejected":0,"latencyTotal":{"0":0,"25":0,"50":12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100": 1560},"requestCount":1376,"rollingCountCollapsedRequests":94516,"rollingCountShortCircuited":0,"latencyExecute":{"0":0,"25":0,"50": 12,"75":48,"90":68,"95":96,"99":192,"99.5":1452,"100":

    1560},"propertyValue_circuitBreakerSleepWindowInMilliseconds":"5000","currentConcurrentExecutionCount": 0,"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests":"10","errorPercentage":0,"rollingCountThreadPoolRejected": 0,"propertyValue_circuitBreakerEnabled":true,"propertyValue_executionIsolationThreadInterruptOnTimeout":true,"propertyValue_request CacheEnabled":true,"rollingCountFallbackRejection":0,"propertyValue_requestLogEnabled":true,"rollingCountSuccess": 1432,"propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests":"10","propertyValue_circuitBreakerErrorThresholdPercentage":"5 0","propertyValue_circuitBreakerForceClosed":false,"name":"GetPredictions","reportingHosts": 4,"propertyValue_executionIsolationThreadPoolKeyOverride":"null","propertyValue_executionIsolationThreadTimeoutInMilliseconds":"100 0"} data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":317,"rollingCountSuccess":327,"reportingHosts":1} data: {“type":"HystrixCommand","name":"GetPredictions","requestCount":1376,"rollingCountSuccess":1432,"reportingHosts":4}
  49. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  50. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  51. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  52. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  53. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] []
  54. /* The following will emit a buffered list as it

    is debounced */ // first we multicast the stream ... using refCount so it handles the subscribe/unsubscribe Observable<Integer> burstStream = intermittentBursts().take(20).publish().refCount(); // then we get the debounced version Observable<Integer> debounced = burstStream.debounce(10, TimeUnit.MILLISECONDS); // then the buffered one that uses the debounced stream to demark window start/stop Observable<List<Integer>> buffered = burstStream.buffer(debounced); // then we subscribe to the buffered stream so it does what we want buffered.toBlocking().forEach(System.out::println); [0, 1, 2] [0, 1, 2] [0, 1, 2, 3, 4, 5, 6] [0, 1, 2, 3, 4] [0, 1] [] https://gist.github.com/benjchristensen/e4524a308456f3c21c0b
  55. Push when consumer keeps up with producer. ! Switch to

    Pull when consumer is slow. ! Bound all* queues.
  56. Push when consumer keeps up with producer. ! Switch to

    Pull when consumer is slow. ! Bound all* queues. *vertically, not horizontally
  57. Observable Subscriber Observable.create(subscriber -> { // emit to subscriber }).subscribe(new

    Subscriber<String>() { ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! });
  58. observable.subscribe Observable.create(subscriber -> { // emit to subscriber }).subscribe(new Subscriber<String>()

    { ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! });
  59. observable.subscribe subscriber.setProducer Observable.create(subscriber -> { subscriber.setProducer(request -> { }); }).subscribe(new

    Subscriber<String>() { ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! });
  60. Observable.create(subscriber -> { subscriber.setProducer(request -> { }); }).subscribe(new Subscriber<String>() {

    ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! }); observable.subscribe subscriber.setProducer producer.request(n)
  61. Observable.create(subscriber -> { subscriber.setProducer(request -> { subscriber.onNext(t); // 5 times

    }); }).subscribe(new Subscriber<String>() { ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { ! } ! }); observable.subscribe subscriber.setProducer producer.request(n) subscriber.onNext(t) …
  62. Observable.create(subscriber -> { subscriber.setProducer(request -> { subscriber.onNext(t); // 5 times

    }); }).subscribe(new Subscriber<String>() { ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { doStuffWith(t); if(requested—- == 0) request(5); } ! }); observable.subscribe subscriber.setProducer producer.request(n) subscriber.onNext(t) … producer.request(n)
  63. Observable.create(subscriber -> { subscriber.setProducer(request -> { … previous logic …

    subscriber.onCompleted(); }); }).subscribe(new Subscriber<String>() { ! public void onStart() { request(5); } ! public void onCompleted() { // do stuff } ! public void onError(Throwable e) { // handle error } ! public void onNext(String t) { doStuffWith(t); if(requested—- == 0) request(5); } ! }); observable.subscribe subscriber.setProducer producer.request(n) subscriber.onNext(t) … producer.request(n) subscriber.onCompleted/onError unsubscribe
  64. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { // do something with the emitted item "n" // request another item: request(1); } });
  65. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { // do something with the emitted item "n" // request another item: request(1); } });
  66. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { // do something with the emitted item "n" // request another item: request(1); } });
  67. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1024); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { enqueue(n); // elsewhere the queue is drained // and request(m) is called } ! });
  68. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1024); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { enqueue(n); // elsewhere the queue is drained // and request(m) is called } ! });
  69. someObservable.subscribe(new Subscriber<T>() { @Override public void onStart() { request(1024); }

    ! @Override public void onCompleted() { // gracefully handle sequence-complete } ! @Override public void onError(Throwable e) { // gracefully handle error } ! @Override public void onNext(T n) { enqueue(n); // elsewhere the queue is drained // and request(m) is called } ! });
  70. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  71. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  72. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  73. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  74. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  75. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  76. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  77. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  78. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  79. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  80. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  81. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  82. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  83. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  84. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  85. MantisJob .source(NetflixSources.moviePlayAttempts()) .stage(playAttempts -> { return playAttempts.groupBy(playAttempt -> { return

    playAttempt.getMovieId(); }) }) .stage(playAttemptsByMovieId -> { playAttemptsByMovieId .window(10,TimeUnit.MINUTES, 1000) // buffer for 10 minutes, or 1000 play attempts .flatMap(windowOfPlayAttempts -> { return windowOfPlayAttempts .reduce(new FailRatioExperiment(playAttemptsByMovieId.getKey()), (experiment, playAttempt) -> { experiment.updateFailRatio(playAttempt); experiment.updateExamples(playAttempt); return experiment; }).doOnNext(experiment -> { logToHistorical("Play attempt experiment", experiment.getId(),experiment); // log for offline analysis }).filter(experiment -> { return experiment.failRatio() >= DYNAMIC_PROP("fail_threshold").get(); }).map(experiment -> { return new FailReport(experiment, runCorrelations(experiment.getExamples())); }).doOnNext(report -> { logToHistorical("Failure report", report.getId(), report); // log for offline analysis }) }) }) .sink(Sinks.emailAlert(report -> { return toEmail(report)})) // anomalies trigger events (simple email here)
  86. ! Reactive Programming in the Netflix API with RxJava http://techblog.netflix.com/2013/02/rxjava-netflix-api.html

    ! Optimizing the Netflix API http://techblog.netflix.com/2013/01/optimizing-netflix-api.html ! Reactive Extensions (Rx) http://www.reactivex.io ! Reactive Streams https://github.com/reactive-streams/reactive-streams ! ! ! ! ! Ben Christensen @benjchristensen RxJava https://github.com/ReactiveX/RxJava @RxJava RxJS http://reactive-extensions.github.io/RxJS/ @ReactiveX jobs.netflix.com