Examples with MantisGroup io.mantisrx.common.MantisGroup used on opensource projects

Example 1 with MantisGroup

use of io.mantisrx.common.MantisGroup in project mantis by Netflix.

the class StageExecutors method executeMantisGroups.

@SuppressWarnings("unchecked")
private static <K, T, R> Observable<Observable<R>> executeMantisGroups(Observable<Observable<MantisGroup<K, T>>> go, final Computation computation, final Context context, final long groupTakeUntil) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>> c = (Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>>) computation;
    return go.lift(new MonitorOperator<>("worker_stage_outer")).map((Func1<Observable<MantisGroup<K, T>>, Observable<R>>) group -> c.call(context, group.lift(new MonitorOperator<>("worker_stage_inner_input"))).lift(new MonitorOperator("worker_stage_inner_output")));
}

Example 2 with MantisGroup

use of io.mantisrx.common.MantisGroup in project mantis by Netflix.

the class StageExecutors method executeMantisGroupsInParallel.

/**
 * @param go
 * @param computation
 *
 * @return untyped to support multiple callers return types
 */
@SuppressWarnings("unchecked")
private static <K, T, R> Observable<Observable<R>> executeMantisGroupsInParallel(Observable<Observable<MantisGroup<K, T>>> go, Computation computation, final Context context, final boolean applyTimeoutToInners, final long timeout, final int concurrency) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>> c = (Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>>) computation;
    if (concurrency == StageConfig.DEFAULT_STAGE_CONCURRENCY) {
        return go.lift(new MonitorOperator<>("worker_stage_outer")).map(observable -> c.call(context, observable.observeOn(Schedulers.computation()).lift(new MonitorOperator<>("worker_stage_inner_input"))).lift(new MonitorOperator<>("worker_stage_inner_output")));
    } else {
        final MantisRxSingleThreadScheduler[] mantisRxSingleThreadSchedulers = new MantisRxSingleThreadScheduler[concurrency];
        RxThreadFactory rxThreadFactory = new RxThreadFactory("MantisRxSingleThreadScheduler-");
        logger.info("creating {} Mantis threads", concurrency);
        for (int i = 0; i < concurrency; i++) {
            mantisRxSingleThreadSchedulers[i] = new MantisRxSingleThreadScheduler(rxThreadFactory);
        }
        return go.lift(new MonitorOperator<>("worker_stage_outer")).map(observable -> observable.groupBy(e -> Math.abs(e.getKeyValue().hashCode()) % concurrency).flatMap(gbo -> c.call(context, gbo.observeOn(mantisRxSingleThreadSchedulers[gbo.getKey().intValue()]).lift(new MonitorOperator<MantisGroup<K, T>>("worker_stage_inner_input"))).lift(new MonitorOperator<R>("worker_stage_inner_output"))));
    }
}

Example 3 with MantisGroup

use of io.mantisrx.common.MantisGroup in project mantis by Netflix.

the class RemoteObservable method createTcpConnectionToGOServer.

private static <K, V> Observable<MantisGroup<K, V>> createTcpConnectionToGOServer(final ConnectToGroupedObservable<K, V> params, final RemoteUnsubscribe remoteUnsubscribe, final RxMetrics metrics, final Action0 connectionDisconnectCallback, Observable<Integer> closeTrigger, final SpscArrayQueue<MantisGroup<?, ?>> inputQueue) {
    final Decoder<K> keyDecoder = params.getKeyDecoder();
    final Decoder<V> valueDecoder = params.getValueDecoder();
    loadFastProperties();
    return RxNetty.createTcpClient(params.getHost(), params.getPort(), new PipelineConfiguratorComposite<RemoteRxEvent, List<RemoteRxEvent>>(new PipelineConfigurator<RemoteRxEvent, List<RemoteRxEvent>>() {

        @Override
        public void configureNewPipeline(ChannelPipeline pipeline) {
            if (enableNettyLogging) {
                // uncomment to enable debug logging
                pipeline.addFirst(new LoggingHandler(LogLevel.ERROR));
            }
            if (enableHeartBeating) {
                pipeline.addLast("idleStateHandler", new IdleStateHandler(10, 2, 0));
                pipeline.addLast("heartbeat", new HeartbeatHandler());
            }
            if (enableCompression) {
                pipeline.addLast("gzipInflater", new JdkZlibEncoder(ZlibWrapper.GZIP));
                pipeline.addLast("gzipDeflater", new JdkZlibDecoder(ZlibWrapper.GZIP));
            }
            // 4 bytes to encode length
            pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
            // max frame = half MB
            pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(maxFrameLength, 0, 4, 0, 4));
        }
    }, new BatchedRxEventPipelineConfigurator())).connect().flatMap(new Func1<ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>>, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
            // send subscribe event to server
            connection.writeAndFlush(RemoteRxEvent.subscribed(params.getName(), params.getSubscribeParameters()));
            remoteUnsubscribe.setConnection(connection);
            return connection.getInput().lift(new DropOperator<RemoteRxEvent>("incoming_" + RemoteObservable.class.getCanonicalName() + "_createTcpConnectionToServerGroups"));
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            // connection completed
            logger.warn("Detected connection completed when trying to connect to host: " + params.getHost() + " port: " + params.getPort());
            connectionDisconnectCallback.call();
        }
    }).onErrorResumeNext(new Func1<Throwable, Observable<RemoteRxEvent>>() {

        @Override
        public Observable<RemoteRxEvent> call(Throwable t1) {
            logger.warn("Detected connection error when trying to connect to host: " + params.getHost() + " port: " + params.getPort(), t1);
            connectionDisconnectCallback.call();
            // complete if error occurs
            return Observable.empty();
        }
    }).takeUntil(closeTrigger).filter(new Func1<RemoteRxEvent, Boolean>() {

        @Override
        public Boolean call(RemoteRxEvent rxEvent) {
            return (rxEvent.getType() == RemoteRxEvent.Type.next);
        }
    }).map(new Func1<RemoteRxEvent, Notification<byte[]>>() {

        @Override
        public Notification<byte[]> call(RemoteRxEvent rxEvent) {
            metrics.incrementNextCount();
            return Notification.createOnNext(rxEvent.getData());
        }
    }).<byte[]>dematerialize().map(new Func1<byte[], MantisGroup<K, V>>() {

        @Override
        public MantisGroup<K, V> call(byte[] bytes) {
            ByteBuffer buff = ByteBuffer.wrap((byte[]) bytes);
            // ignore notification type in key selector
            buff.get();
            int keyLength = buff.getInt();
            byte[] key = new byte[keyLength];
            buff.get(key);
            K keyVal = keyDecoder.decode(key);
            V value = null;
            buff = ByteBuffer.wrap((byte[]) bytes);
            byte notificationType = buff.get();
            if (notificationType == 1) {
                // int keyLength = buff.getInt();
                int end = buff.limit();
                int dataLength = end - 4 - 1 - keyLength;
                byte[] valueBytes = new byte[dataLength];
                buff.position(4 + 1 + keyLength);
                buff.get(valueBytes, 0, dataLength);
                value = valueDecoder.decode(valueBytes);
            } else {
                throw new RuntimeException("Notification encoding not support: " + notificationType);
            }
            return new MantisGroup<K, V>(keyVal, value);
        }
    }).doOnEach(new Observer<MantisGroup<K, V>>() {

        @Override
        public void onCompleted() {
            logger.info("RemoteRxEvent, name: " + params.getName() + " onCompleted()");
        }

        @Override
        public void onError(Throwable e) {
            logger.error("RemoteRxEvent, name: " + params.getName() + " onError()", e);
        }

        @Override
        public void onNext(MantisGroup<K, V> group) {
            if (logger.isDebugEnabled()) {
                logger.debug("RemoteRxEvent, name: " + params.getName() + " new key: " + group.getKeyValue());
            }
        }
    });
}

Example 4 with MantisGroup

use of io.mantisrx.common.MantisGroup in project mantis by Netflix.

the class StageExecutors method executeIntermediate.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T, R> void executeIntermediate(WorkerConsumer consumer, final StageConfig<T, R> stage, WorkerPublisher publisher, final Context context) {
    if (consumer == null) {
        throw new IllegalArgumentException("consumer cannot be null");
    }
    if (stage == null) {
        throw new IllegalArgumentException("stage cannot be null");
    }
    if (publisher == null) {
        throw new IllegalArgumentException("producer cannot be null");
    }
    Observable<?> toSink = null;
    if (stage instanceof ScalarToScalar) {
        ScalarToScalar scalarStage = (ScalarToScalar) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToScalarStage(scalarStage, source, context);
    } else if (stage instanceof ScalarToKey) {
        ScalarToKey scalarStage = (ScalarToKey) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToKeyStage(scalarStage, source, context);
    } else // NJ
    if (stage instanceof ScalarToGroup) {
        ScalarToGroup scalarStage = (ScalarToGroup) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToGroupStage(scalarStage, source, context);
    } else if (stage instanceof KeyToKey) {
        KeyToKey keyToKey = (KeyToKey) stage;
        Observable<Observable<GroupedObservable<String, T>>> source = consumer.start(keyToKey);
        toSink = setupKeyToKeyStage(keyToKey, source, context);
    } else if (stage instanceof GroupToGroup) {
        GroupToGroup groupToGroup = (GroupToGroup) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(groupToGroup);
        toSink = setupGroupToGroupStage(groupToGroup, source, context);
    } else if (stage instanceof KeyToScalar) {
        KeyToScalar scalarToKey = (KeyToScalar) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(scalarToKey);
        toSink = setupKeyToScalarStage(scalarToKey, source, context);
    } else if (stage instanceof GroupToScalar) {
        GroupToScalar groupToScalar = (GroupToScalar) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(groupToScalar);
        toSink = setupGroupToScalarStage(groupToScalar, source, context);
    }
    publisher.start(stage, toSink);
}

Example 5 with MantisGroup

use of io.mantisrx.common.MantisGroup in project mantis by Netflix.

the class ObservableTrigger method mantisGroupTrigger.

private static <K, V> PushTrigger<KeyValuePair<K, V>> mantisGroupTrigger(final String name, final Observable<MantisGroup<K, V>> o, final Action0 doOnComplete, final Action1<Throwable> doOnError, final long groupExpirySeconds, final Func1<K, byte[]> keyEncoder, final HashFunction hashFunction) {
    final AtomicReference<Subscription> subRef = new AtomicReference<>();
    final Gauge subscriptionActive;
    Metrics metrics = new Metrics.Builder().name("ObservableTrigger_" + name).addGauge("subscriptionActive").build();
    subscriptionActive = metrics.getGauge("subscriptionActive");
    Action1<MonitoredQueue<KeyValuePair<K, V>>> doOnStart = new Action1<MonitoredQueue<KeyValuePair<K, V>>>() {

        @Override
        public void call(final MonitoredQueue<KeyValuePair<K, V>> queue) {
            subRef.set(o.doOnSubscribe(() -> {
                logger.info("Subscription is ACTIVE for observable trigger with name: " + name);
                subscriptionActive.set(1);
            }).doOnUnsubscribe(() -> {
                logger.info("Subscription is INACTIVE for observable trigger with name: " + name);
                subscriptionActive.set(0);
            }).map((MantisGroup<K, V> data) -> {
                final byte[] keyBytes = keyEncoder.call(data.getKeyValue());
                final long keyBytesHashed = hashFunction.computeHash(keyBytes);
                return (new KeyValuePair<K, V>(keyBytesHashed, keyBytes, data.getValue()));
            }).subscribe((KeyValuePair<K, V> data) -> {
                queue.write(data);
            }, (Throwable e) -> {
                logger.warn("Observable used to push data errored, on server with name: " + name, e);
                if (doOnError != null) {
                    doOnError.call(e);
                }
            }, () -> {
                logger.info("Observable used to push data completed, on server with name: " + name);
                if (doOnComplete != null) {
                    doOnComplete.call();
                }
            }));
        }
    };
    Action1<MonitoredQueue<KeyValuePair<K, V>>> doOnStop = new Action1<MonitoredQueue<KeyValuePair<K, V>>>() {

        @Override
        public void call(MonitoredQueue<KeyValuePair<K, V>> t1) {
            if (subRef.get() != null) {
                logger.warn("Connections from next stage has dropped to 0. Do not propagate unsubscribe");
            // subRef.get().unsubscribe();
            }
        }
    };
    return new PushTrigger<>(doOnStart, doOnStop, metrics);
}