Examples with GroupedObservable rx.observables.GroupedObservable used on opensource projects

Example 16 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class RemoteObservable method createTcpConnectionToServer.

private static <K, V> Observable<GroupedObservable<K, V>> createTcpConnectionToServer(final ConnectToGroupedObservable<K, V> params, final RemoteUnsubscribe remoteUnsubscribe, final RxMetrics metrics, final Action0 connectionDisconnectCallback, Observable<Integer> closeTrigger) {
    final Decoder<K> keyDecoder = params.getKeyDecoder();
    final Decoder<V> valueDecoder = params.getValueDecoder();
    loadFastProperties();
    return RxNetty.createTcpClient(params.getHost(), params.getPort(), new PipelineConfiguratorComposite<RemoteRxEvent, List<RemoteRxEvent>>(new PipelineConfigurator<RemoteRxEvent, List<RemoteRxEvent>>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            if (enableNettyLogging) {
                // uncomment to enable debug logging
                pipeline.addFirst(new LoggingHandler(LogLevel.ERROR));
            }
            if (enableHeartBeating) {
                pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
                pipeline.addLast("heartbeat", new HeartbeatHandler());
            }
            if (enableCompression) {
                pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
                pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            }
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(maxFrameLength, 0, 4, 0, 4));
        }
    }, new BatchedRxEventPipelineConfigurator())).connect().flatMap(new Func1<ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>>, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
            // send subscribe event to server
            connection.writeAndFlush(RemoteRxEvent.subscribed(params.getName(), params.getSubscribeParameters()));
            remoteUnsubscribe.setConnection(connection);
            return connection.getInput().lift(new DropOperator<RemoteRxEvent>("incoming_" + RemoteObservable.class.getCanonicalName() + "_createTcpConnectionToServerGroups"));
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            // connection completed
            logger.warn("Detected connection completed when trying to connect to host: " + params.getHost() + " port: " + params.getPort());
            connectionDisconnectCallback.call();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(Throwable t1) {
            logger.warn("Detected connection error when trying to connect to host: " + params.getHost() + " port: " + params.getPort(), t1);
            connectionDisconnectCallback.call();
            // complete if error occurs
            return Observable.empty();
        }
    }).takeUntil(closeTrigger).map(new Func1<RemoteRxEvent, Notification<byte[]>>() {

        @Override
        public Notification<byte[]> call(RemoteRxEvent rxEvent) {
            if (rxEvent.getType() == RemoteRxEvent.Type.next) {
                metrics.incrementNextCount();
                return Notification.createOnNext(rxEvent.getData());
            } else if (rxEvent.getType() == RemoteRxEvent.Type.error) {
                metrics.incrementErrorCount();
                return Notification.createOnError(fromBytesToThrowable(rxEvent.getData()));
            } else if (rxEvent.getType() == RemoteRxEvent.Type.completed) {
                metrics.incrementCompletedCount();
                return Notification.createOnCompleted();
            } else {
                throw new RuntimeException("RemoteRxEvent of type:" + rxEvent.getType() + ", not supported.");
            }
        }
    }).<byte[]>dematerialize().groupBy(new Func1<byte[], K>() {

        @Override
        public K call(byte[] bytes) {
            ByteBuffer buff = ByteBuffer.wrap((byte[]) bytes);
            // ignore notification type in key selector
            buff.get();
            int keyLength = buff.getInt();
            byte[] key = new byte[keyLength];
            buff.get(key);
            return keyDecoder.decode(key);
        }
    }, new Func1<byte[], Notification<V>>() {

        @Override
        public Notification<V> call(byte[] bytes) {
            ByteBuffer buff = ByteBuffer.wrap((byte[]) bytes);
            byte notificationType = buff.get();
            if (notificationType == 1) {
                int keyLength = buff.getInt();
                int end = buff.limit();
                int dataLength = end - 4 - 1 - keyLength;
                byte[] valueBytes = new byte[dataLength];
                buff.position(4 + 1 + keyLength);
                buff.get(valueBytes, 0, dataLength);
                V value = valueDecoder.decode(valueBytes);
                return Notification.createOnNext(value);
            } else if (notificationType == 2) {
                return Notification.createOnCompleted();
            } else if (notificationType == 3) {
                int keyLength = buff.getInt();
                int end = buff.limit();
                int dataLength = end - 4 - 1 - keyLength;
                byte[] errorBytes = new byte[dataLength];
                buff.position(4 + 1 + keyLength);
                buff.get(errorBytes, 0, dataLength);
                return Notification.createOnError(fromBytesToThrowable(errorBytes));
            } else {
                throw new RuntimeException("Notification encoding not support: " + notificationType);
            }
        }
    }).map(new Func1<GroupedObservable<K, Notification<V>>, GroupedObservable<K, V>>() {

        @Override
        public GroupedObservable<K, V> call(GroupedObservable<K, Notification<V>> group) {
            return GroupedObservableUtils.createGroupedObservable(group.getKey(), group.<V>dematerialize());
        }
    }).doOnEach(new Observer<GroupedObservable<K, V>>() {

        @Override
        public void onCompleted() {
            logger.info("RemoteRxEvent, name: {} onCompleted()", params.getName());
        }

        @Override
        public void onError(Throwable e) {
            logger.error("RemoteRxEvent, name: {} onError()", params.getName(), e);
        }

        @Override
        public void onNext(GroupedObservable<K, V> group) {
            if (logger.isDebugEnabled()) {
                logger.debug("RemoteRxEvent, name: {} new key: {}", params.getName(), group.getKey());
            }
        }
    });
}

Example 17 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class ServeGroupedObservable method applySlottingSideEffectToObservable.

private void applySlottingSideEffectToObservable(Observable<Observable<GroupedObservable<String, V>>> o, final Observable<Integer> minConnectionsToSubscribe) {
    final AtomicInteger currentMinConnectionsToSubscribe = new AtomicInteger();
    minConnectionsToSubscribe.subscribe(new Action1<Integer>() {

        @Override
        public void call(Integer t1) {
            currentMinConnectionsToSubscribe.set(t1);
        }
    });
    Observable<Observable<List<Group<String, V>>>> listOfGroups = o.map(new Func1<Observable<GroupedObservable<String, V>>, Observable<List<Group<String, V>>>>() {

        @Override
        public Observable<List<Group<String, V>>> call(Observable<GroupedObservable<String, V>> og) {
            return og.flatMap(new Func1<GroupedObservable<String, V>, Observable<List<Group<String, V>>>>() {

                @Override
                public Observable<List<Group<String, V>>> call(final GroupedObservable<String, V> group) {
                    final byte[] keyBytes = keyEncoder.encode(group.getKey());
                    final String keyValue = group.getKey();
                    return group.doOnUnsubscribe(new Action0() {

                        @Override
                        public void call() {
                            // logger.info("Expiring group stage in serveGroupedObservable " + group.getKey());
                            groupsExpiredCounter.increment();
                        }
                    }).timeout(expiryInSecs, TimeUnit.SECONDS, (Observable<? extends V>) Observable.empty()).materialize().lift(new DisableBackPressureOperator<Notification<V>>()).buffer(groupBufferTimeMSec, TimeUnit.MILLISECONDS).filter(new Func1<List<Notification<V>>, Boolean>() {

                        @Override
                        public Boolean call(List<Notification<V>> t1) {
                            return t1 != null && !t1.isEmpty();
                        }
                    }).map(new Func1<List<Notification<V>>, List<Group<String, V>>>() {

                        @Override
                        public List<Group<String, V>> call(List<Notification<V>> notifications) {
                            List<Group<String, V>> groups = new ArrayList<>(notifications.size());
                            for (Notification<V> notification : notifications) {
                                groups.add(new Group<String, V>(keyValue, keyBytes, notification));
                            }
                            return groups;
                        }
                    });
                }
            });
        }
    });
    final Observable<List<Group<String, V>>> withSideEffects = Observable.merge(listOfGroups).doOnEach(new Observer<List<Group<String, V>>>() {

        @Override
        public void onCompleted() {
            slottingStrategy.completeAllConnections();
        }

        @Override
        public void onError(Throwable e) {
            e.printStackTrace();
            slottingStrategy.errorAllConnections(e);
        }

        @Override
        public void onNext(List<Group<String, V>> listOfGroups) {
            for (Group<String, V> group : listOfGroups) {
                slottingStrategy.writeOnSlot(group.getKeyBytes(), group);
            }
        }
    });
    final MutableReference<Subscription> subscriptionRef = new MutableReference<>();
    final AtomicInteger connectionCount = new AtomicInteger(0);
    final AtomicBoolean isSubscribed = new AtomicBoolean();
    slottingStrategy.registerDoOnEachConnectionAdded(new Action0() {

        @Override
        public void call() {
            Integer minNeeded = currentMinConnectionsToSubscribe.get();
            Integer current = connectionCount.incrementAndGet();
            if (current >= minNeeded) {
                if (isSubscribed.compareAndSet(false, true)) {
                    logger.info("MinConnectionsToSubscribe: " + minNeeded + ", has been met, subscribing to observable, current connection count: " + current);
                    subscriptionRef.setValue(withSideEffects.subscribe());
                }
            } else {
                logger.info("MinConnectionsToSubscribe: " + minNeeded + ", has NOT been met, current connection count: " + current);
            }
        }
    });
    slottingStrategy.registerDoAfterLastConnectionRemoved(new Action0() {

        @Override
        public void call() {
            subscriptionRef.getValue().unsubscribe();
            logger.info("All connections deregistered, unsubscribed to observable, resetting current connection count: 0");
            connectionCount.set(0);
            isSubscribed.set(false);
        }
    });
}

Example 18 with GroupedObservable

use of rx.observables.GroupedObservable in project pinpoint by naver.

the class GroupedObservableTestRunner method groupedObservable.

public void groupedObservable() throws Exception {
    int numMessageChunks = 2;
    final List<String> messages = new ArrayList<String>();
    final List<String> messageChunk = Arrays.asList("Hello", "World");
    for (int i = 0; i < numMessageChunks; i++) {
        messages.addAll(messageChunk);
    }
    final CountDownLatch completeLatch = new CountDownLatch(1);
    final List<String> helloMessages = Collections.synchronizedList(new ArrayList<String>());
    final List<String> worldMessages = Collections.synchronizedList(new ArrayList<String>());
    Observable<GroupedObservable<String, String>> grouped = echoesService.echo(messages).subscribeOn(Schedulers.computation()).groupBy(new Func1<String, String>() {

        @Override
        public String call(String s) {
            return s;
        }
    });
    grouped.subscribe(new Action1<GroupedObservable<String, String>>() {

        @Override
        public void call(final GroupedObservable<String, String> groupedObservable) {
            String key = groupedObservable.getKey();
            if (key.equals("Hello")) {
                groupedObservable.subscribe(new Action1<String>() {

                    @Override
                    public void call(String s) {
                        helloMessages.add(s);
                    }
                });
            } else if (key.equals("World")) {
                groupedObservable.subscribe(new Action1<String>() {

                    @Override
                    public void call(String s) {
                        worldMessages.add(s);
                    }
                });
            }
        }
    }, new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            completeLatch.countDown();
        }
    }, new Action0() {

        @Override
        public void call() {
            completeLatch.countDown();
        }
    });
    completeLatch.await(500L, TimeUnit.MILLISECONDS);
    Assert.assertEquals(messages.size() / messageChunk.size(), helloMessages.size());
    Assert.assertEquals(messages.size() / messageChunk.size(), worldMessages.size());
    for (String helloMessage : helloMessages) {
        Assert.assertEquals("Hello", helloMessage);
    }
    for (String worldMessage : worldMessages) {
        Assert.assertEquals("World", worldMessage);
    }
    TestHelper.awaitForSpanDataFlush();
    PluginTestVerifier verifier = PluginTestVerifierHolder.getInstance();
    verifier.awaitTrace(event(ServiceType.ASYNC.getName(), "Asynchronous Invocation"), 20, 3000);
    verifier.printCache();
    // Skip rx java internal traces as they differ between versions and it's too much work to split the tests.
    // Instead, we can verify them indirectly by checking if user methods have been traced.
    verifier.ignoreServiceType("RX_JAVA_INTERNAL");
    Method groupByMethod = Observable.class.getDeclaredMethod("groupBy", Func1.class);
    verifier.verifyTrace(event("RX_JAVA", groupByMethod));
    Method subscribeMethod = Observable.class.getDeclaredMethod("subscribe", Action1.class, Action1.class, Action0.class);
    verifier.verifyTrace(event("RX_JAVA", subscribeMethod));
    // event - RX_JAVA_INTERNAL some form of Worker.schedule(Action0)
    verifier.verifyTrace(event(ServiceType.ASYNC.getName(), "Asynchronous Invocation"));
    // event - RX_JAVA_INTERNAL some form of Action0 implementation's call() inside OperatorSubscribeOn that gets scheduled
    Method echoMethod = EchoRepository.class.getDeclaredMethod("echo", String.class);
    Method groupedObservableSubscribeMethod = Observable.class.getDeclaredMethod("subscribe", Action1.class);
    for (int i = 0; i < numMessageChunks; i++) {
        for (int j = 0; j < messageChunk.size(); j++) {
            verifier.verifyTrace(event(ServiceType.INTERNAL_METHOD.getName(), echoMethod));
            if (i == 0) {
                verifier.verifyTrace(event("RX_JAVA", groupedObservableSubscribeMethod));
            }
        }
    }
}

Example 19 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class StageExecutors method executeIntermediate.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T, R> void executeIntermediate(WorkerConsumer consumer, final StageConfig<T, R> stage, WorkerPublisher publisher, final Context context) {
    if (consumer == null) {
        throw new IllegalArgumentException("consumer cannot be null");
    }
    if (stage == null) {
        throw new IllegalArgumentException("stage cannot be null");
    }
    if (publisher == null) {
        throw new IllegalArgumentException("producer cannot be null");
    }
    Observable<?> toSink = null;
    if (stage instanceof ScalarToScalar) {
        ScalarToScalar scalarStage = (ScalarToScalar) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToScalarStage(scalarStage, source, context);
    } else if (stage instanceof ScalarToKey) {
        ScalarToKey scalarStage = (ScalarToKey) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToKeyStage(scalarStage, source, context);
    } else // NJ
    if (stage instanceof ScalarToGroup) {
        ScalarToGroup scalarStage = (ScalarToGroup) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToGroupStage(scalarStage, source, context);
    } else if (stage instanceof KeyToKey) {
        KeyToKey keyToKey = (KeyToKey) stage;
        Observable<Observable<GroupedObservable<String, T>>> source = consumer.start(keyToKey);
        toSink = setupKeyToKeyStage(keyToKey, source, context);
    } else if (stage instanceof GroupToGroup) {
        GroupToGroup groupToGroup = (GroupToGroup) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(groupToGroup);
        toSink = setupGroupToGroupStage(groupToGroup, source, context);
    } else if (stage instanceof KeyToScalar) {
        KeyToScalar scalarToKey = (KeyToScalar) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(scalarToKey);
        toSink = setupKeyToScalarStage(scalarToKey, source, context);
    } else if (stage instanceof GroupToScalar) {
        GroupToScalar groupToScalar = (GroupToScalar) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(groupToScalar);
        toSink = setupGroupToScalarStage(groupToScalar, source, context);
    }
    publisher.start(stage, toSink);
}

Example 20 with GroupedObservable

use of rx.observables.GroupedObservable in project mantis by Netflix.

the class OperatorGroupByTest method testError.

@Test
public void testError() {
    Observable<String> sourceStrings = Observable.just("one", "two", "three", "four", "five", "six");
    Observable<String> errorSource = Observable.error(new RuntimeException("forced failure"));
    Observable<String> source = Observable.concat(sourceStrings, errorSource);
    Observable<GroupedObservable<Integer, String>> grouped = source.lift(new OperatorGroupBy<String, Integer, String>(length));
    final AtomicInteger groupCounter = new AtomicInteger();
    final AtomicInteger eventCounter = new AtomicInteger();
    final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
    grouped.flatMap(new Func1<GroupedObservable<Integer, String>, Observable<String>>() {

        @Override
        public Observable<String> call(final GroupedObservable<Integer, String> o) {
            groupCounter.incrementAndGet();
            return o.map(new Func1<String, String>() {

                @Override
                public String call(String v) {
                    return "Event => key: " + o.getKey() + " value: " + v;
                }
            });
        }
    }).subscribe(new Subscriber<String>() {

        @Override
        public void onCompleted() {
        }

        @Override
        public void onError(Throwable e) {
            e.printStackTrace();
            error.set(e);
        }

        @Override
        public void onNext(String v) {
            eventCounter.incrementAndGet();
            System.out.println(v);
        }
    });
    assertEquals(3, groupCounter.get());
    assertEquals(6, eventCounter.get());
    assertNotNull(error.get());
}