Examples with GroupedObservable rx.observables.GroupedObservable used on opensource projects

Example 11 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class StageExecutors method executeGroupsInParallel.

@SuppressWarnings("unchecked")
private static <K, T, R> Observable<Observable<R>> executeGroupsInParallel(Observable<GroupedObservable<K, T>> go, final Computation computation, final Context context, final long groupTakeUntil) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, GroupedObservable<K, T>, Observable<R>> c = (Func2<Context, GroupedObservable<K, T>, Observable<R>>) computation;
    return go.lift(new MonitorOperator<>("worker_stage_outer")).map((Func1<GroupedObservable<K, T>, Observable<R>>) group -> c.call(context, GroupedObservableUtils.createGroupedObservable(group.getKey(), group.doOnUnsubscribe(() -> {
        if (groupsExpiredCounter != null)
            groupsExpiredCounter.increment();
    }).timeout(groupTakeUntil, TimeUnit.SECONDS, (Observable<? extends T>) Observable.empty()).subscribeOn(Schedulers.computation()).lift(new MonitorOperator<T>("worker_stage_inner_input")))).lift(new MonitorOperator("worker_stage_inner_output")));
}

Example 12 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class StageExecutorsGroupByTest method testExecuteSource.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSource() {
    TestGroupByJob provider = new TestGroupByJob();
    Job<Pair> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    int serverPort = portSelector.acquirePort();
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(serverPort, null, Observable.just(1), null);
    // execute source
    BehaviorSubject<Integer> workersInStageOneObservable = BehaviorSubject.create(1);
    StageExecutors.executeSource(0, job.getSource(), stages.get(0), producer, new Context(), workersInStageOneObservable);
    ConnectToGroupedObservable<String, Integer> config = new ConnectToGroupedObservable.Builder<String, Integer>().slotId("0").host("localhost").port(serverPort).keyDecoder(Codecs.string()).valueDecoder(Codecs.integer()).build();
    Iterator<GroupedObservable<String, Integer>> iter = RemoteObservable.connect(config).getObservable().toBlocking().getIterator();
    Assert.assertTrue(iter.hasNext());
    // verify numbers are grouped by even/odd
    // even is first due to zero
    GroupedObservable<String, Integer> even = iter.next();
    Assert.assertEquals("even", even.getKey());
    Iterator<Integer> evenIter = even.toBlocking().getIterator();
    Assert.assertEquals(0, evenIter.next().intValue());
    Assert.assertEquals(2, evenIter.next().intValue());
    Assert.assertEquals(4, evenIter.next().intValue());
    Assert.assertEquals(6, evenIter.next().intValue());
    GroupedObservable<String, Integer> odd = iter.next();
    Assert.assertEquals("odd", odd.getKey());
    Iterator<Integer> oddIter = odd.toBlocking().getIterator();
    Assert.assertEquals(1, oddIter.next().intValue());
    Assert.assertEquals(3, oddIter.next().intValue());
    Assert.assertEquals(5, oddIter.next().intValue());
    Assert.assertEquals(7, oddIter.next().intValue());
    // should only have two groups
    Assert.assertEquals(false, iter.hasNext());
}

Example 13 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class StageExecutorsGroupByTest method testExecuteIntermediatStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
    // Note, this test has a timing issue, client starts
    // sending data before server is ready, resulting
    // in a RST (connection reset by peer)
    TestGroupByJob provider = new TestGroupByJob();
    Job<Pair> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int publishPort = portSelector.acquirePort();
    final int consumerPort = portSelector.acquirePort();
    Observable<Observable<GroupedObservable<String, Integer>>> go = Observable.just(Observable.range(0, 10).groupBy(new Func1<Integer, String>() {

        @Override
        public String call(Integer t1) {
            if ((t1 % 2) == 0) {
                return "even";
            } else {
                return "odd";
            }
        }
    }));
    // mimic previous stage with a server
    ServeGroupedObservable<String, Integer> config = new ServeGroupedObservable.Builder<String, Integer>().keyEncoder(Codecs.string()).valueEncoder(Codecs.integer()).observable(go).build();
    RemoteRxServer server = new RemoteRxServer.Builder().addObservable(config).port(consumerPort).build();
    server.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "0")));
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
    // execute source
    StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
    ConnectToGroupedObservable<String, Integer> connectConfig = new ConnectToGroupedObservable.Builder<String, Integer>().host("localhost").port(publishPort).keyDecoder(Codecs.string()).valueDecoder(Codecs.integer()).build();
    Iterator<GroupedObservable<String, Integer>> iter = RemoteObservable.connect(connectConfig).getObservable().toBlocking().getIterator();
    // verify numbers are grouped by even/odd
    // even is first due to zero
    GroupedObservable<String, Integer> even = iter.next();
    Assert.assertEquals("even", even.getKey());
    Iterator<Integer> evenIter = even.toBlocking().getIterator();
    Assert.assertEquals(0, evenIter.next().intValue());
    Assert.assertEquals(4, evenIter.next().intValue());
    Assert.assertEquals(16, evenIter.next().intValue());
    Assert.assertEquals(36, evenIter.next().intValue());
    GroupedObservable<String, Integer> odd = iter.next();
    Assert.assertEquals("odd", odd.getKey());
    Iterator<Integer> oddIter = odd.toBlocking().getIterator();
    Assert.assertEquals(1, oddIter.next().intValue());
    Assert.assertEquals(9, oddIter.next().intValue());
    Assert.assertEquals(25, oddIter.next().intValue());
    Assert.assertEquals(49, oddIter.next().intValue());
    // should only have two groups
    Assert.assertEquals(false, iter.hasNext());
}

Example 14 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class ObservableTrigger method groupTrigger.

private static <K, V> PushTrigger<KeyValuePair<K, V>> groupTrigger(final String name, final Observable<GroupedObservable<K, V>> o, final Action0 doOnComplete, final Action1<Throwable> doOnError, final long groupExpirySeconds, final Func1<K, byte[]> keyEncoder, final HashFunction hashFunction) {
    final AtomicReference<Subscription> subRef = new AtomicReference<>();
    final Gauge subscriptionActive;
    Metrics metrics = new Metrics.Builder().name("ObservableTrigger_" + name).addGauge("subscriptionActive").build();
    subscriptionActive = metrics.getGauge("subscriptionActive");
    Action1<MonitoredQueue<KeyValuePair<K, V>>> doOnStart = new Action1<MonitoredQueue<KeyValuePair<K, V>>>() {

        @Override
        public void call(final MonitoredQueue<KeyValuePair<K, V>> queue) {
            subRef.set(o.observeOn(Schedulers.computation()).doOnSubscribe(() -> {
                logger.info("Subscription is ACTIVE for observable trigger with name: " + name);
                subscriptionActive.set(1);
            }).doOnUnsubscribe(() -> {
                logger.info("Subscription is INACTIVE for observable trigger with name: " + name);
                subscriptionActive.set(0);
            }).flatMap((final GroupedObservable<K, V> group) -> {
                final byte[] keyBytes = keyEncoder.call(group.getKey());
                final long keyBytesHashed = hashFunction.computeHash(keyBytes);
                return group.timeout(groupExpirySeconds, TimeUnit.SECONDS, (Observable<? extends V>) Observable.empty()).lift(new DisableBackPressureOperator<V>()).buffer(250, TimeUnit.MILLISECONDS).filter((List<V> t1) -> t1 != null && !t1.isEmpty()).map((List<V> list) -> {
                    List<KeyValuePair<K, V>> keyPairList = new ArrayList<>(list.size());
                    for (V data : list) {
                        keyPairList.add(new KeyValuePair<K, V>(keyBytesHashed, keyBytes, data));
                    }
                    return keyPairList;
                });
            }).subscribe((List<KeyValuePair<K, V>> list) -> {
                for (KeyValuePair<K, V> data : list) {
                    queue.write(data);
                }
            }, (Throwable e) -> {
                logger.warn("Observable used to push data errored, on server with name: " + name, e);
                if (doOnError != null) {
                    doOnError.call(e);
                }
            }, () -> {
                logger.info("Observable used to push data completed, on server with name: " + name);
                if (doOnComplete != null) {
                    doOnComplete.call();
                }
            }));
        }
    };
    Action1<MonitoredQueue<KeyValuePair<K, V>>> doOnStop = new Action1<MonitoredQueue<KeyValuePair<K, V>>>() {

        @Override
        public void call(MonitoredQueue<KeyValuePair<K, V>> t1) {
            if (subRef.get() != null) {
                logger.warn("Connections from next stage has dropped to 0. Do not propagate unsubscribe");
            // subRef.get().unsubscribe();
            }
        }
    };
    return new PushTrigger<>(doOnStart, doOnStop, metrics);
}

Example 15 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class DynamicConnectionSet method observables.

public Observable<Observable<T>> observables() {
    return endpointInjector.deltas().doOnCompleted(() -> logger.info("onComplete on injector deltas")).doOnError(t -> logger.error("caught unexpected error {}", t.getMessage(), t)).doOnSubscribe(new Action0() {

        @Override
        public void call() {
            logger.info("Subscribing, clearing active connection set");
            resetActiveConnections();
        }
    }).groupBy(new Func1<EndpointChange, String>() {

        @Override
        public String call(EndpointChange t1) {
            return Endpoint.uniqueHost(t1.getEndpoint().getHost(), t1.getEndpoint().getPort(), t1.getEndpoint().getSlotId());
        }
    }).flatMap(new Func1<GroupedObservable<String, EndpointChange>, Observable<Observable<T>>>() {

        @Override
        public Observable<Observable<T>> call(final GroupedObservable<String, EndpointChange> group) {
            final PublishSubject<Integer> closeConnectionTrigger = PublishSubject.create();
            return group.doOnNext(new Action1<EndpointChange>() {

                @Override
                public void call(EndpointChange change) {
                    // side effect to force complete
                    if (EndpointChange.Type.complete == change.getType() && activeConnectionsContains(group.getKey(), change.getEndpoint())) {
                        logger.info("Received complete request, removing connection from active set, " + change.getEndpoint().getHost() + " port: " + change.getEndpoint().getPort() + " id: " + change.getEndpoint().getSlotId());
                        forceCompletedConnections.increment();
                        removeConnection(group.getKey(), change.getEndpoint());
                        closeConnectionTrigger.onNext(1);
                    }
                }
            }).filter(new Func1<EndpointChange, Boolean>() {

                @Override
                public Boolean call(EndpointChange change) {
                    // active connection check is to ensure
                    // latent adds are not allowed.  This can
                    // occur if dynamicConnection set
                    // and reconciliator are chained together.
                    // Reconciliator will "see" changes
                    // before dynamic connection set add will
                    // assume connection is missing from set
                    boolean contains = activeConnectionsContains(group.getKey(), change.getEndpoint());
                    if (contains) {
                        logger.info("Skipping latent add for endpoint, already in active set: " + change);
                    }
                    return EndpointChange.Type.add == change.getType() && !contains;
                }
            }).map(new Func1<EndpointChange, Observable<T>>() {

                @Override
                public Observable<T> call(final EndpointChange toAdd) {
                    logger.info("Received add request, adding connection to active set, " + toAdd.getEndpoint().getHost() + " port: " + toAdd.getEndpoint().getPort() + ", with client id: " + toAdd.getEndpoint().getSlotId());
                    addConnection(group.getKey(), toAdd.getEndpoint());
                    Action0 disconnectCallback = new Action0() {

                        @Override
                        public void call() {
                            int timeToWait = random.nextInt((maxTimeoutOnUnexpectedTerminateSec - minTimeoutOnUnexpectedTerminateSec) + 1) + minTimeoutOnUnexpectedTerminateSec;
                            logger.info("Connection disconnected, waiting " + timeToWait + " seconds before removing from active set of connections: " + toAdd);
                            Observable.timer(timeToWait, TimeUnit.SECONDS).doOnCompleted(new Action0() {

                                @Override
                                public void call() {
                                    logger.warn("Removing connection from active set, " + toAdd);
                                    closedConnections.increment();
                                    removeConnection(group.getKey(), toAdd.getEndpoint());
                                }
                            }).subscribe();
                        }
                    };
                    RemoteRxConnection<T> connection = toObservableFunc.call(toAdd.getEndpoint(), disconnectCallback, closeConnectionTrigger);
                    return connection.getObservable().doOnCompleted(toAdd.getEndpoint().getCompletedCallback()).doOnError(toAdd.getEndpoint().getErrorCallback());
                }
            });
        }
    });
}