Examples with KeyToKey io.mantisrx.runtime.KeyToKey used on opensource projects

Example 1 with KeyToKey

use of io.mantisrx.runtime.KeyToKey in project mantis by Netflix.

the class WorkerPublisherRemoteObservable method start.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void start(final StageConfig<T, R> stage, Observable<Observable<R>> toServe) {
    RemoteRxServer.Builder serverBuilder = new RemoteRxServer.Builder();
    if (stage instanceof KeyToKey || stage instanceof ScalarToKey || stage instanceof ScalarToGroup || stage instanceof GroupToGroup) {
        if (runNewW2WserverGroups(jobName)) {
            logger.info("Modern server setup for name: " + name + " type: Keyedstage");
            long expiryTimeInSecs = Long.MAX_VALUE;
            if (stage instanceof KeyToKey) {
                expiryTimeInSecs = ((KeyToKey) stage).getKeyExpireTimeSeconds();
            } else if (stage instanceof ScalarToKey) {
                expiryTimeInSecs = ((ScalarToKey) stage).getKeyExpireTimeSeconds();
            }
            Func1<R, byte[]> valueEncoder = new Func1<R, byte[]>() {

                @Override
                public byte[] call(R t1) {
                    return stage.getOutputCodec().encode(t1);
                }
            };
            Func1<String, byte[]> keyEncoder = new Func1<String, byte[]>() {

                @Override
                public byte[] call(String t1) {
                    return Codecs.string().encode(t1);
                }
            };
            ServerConfig<KeyValuePair<String, R>> config = new ServerConfig.Builder<KeyValuePair<String, R>>().name(name).port(serverPort).metricsRegistry(MetricsRegistry.getInstance()).numQueueConsumers(numConsumerThreads()).maxChunkSize(maxChunkSize()).maxChunkTimeMSec(maxChunkTimeMSec()).bufferCapacity(bufferCapacity()).useSpscQueue(useSpsc()).router(Routers.consistentHashingLegacyTcpProtocol(jobName, keyEncoder, valueEncoder)).build();
            // remove type
            Observable go = toServe;
            if (stage instanceof ScalarToGroup || stage instanceof GroupToGroup) {
                final LegacyTcpPushServer<KeyValuePair<String, R>> modernServer = PushServers.infiniteStreamLegacyTcpNestedMantisGroup(config, go, expiryTimeInSecs, keyEncoder, io.reactivex.mantis.network.push.HashFunctions.ketama());
                modernServer.start();
                // support legacy server interface
                this.server = new RemoteRxServer() {

                    @Override
                    public void start() {
                    }

                    @Override
                    public void startAndWait() {
                    }

                    @Override
                    public void shutdown() {
                        modernServer.shutdown();
                    }

                    @Override
                    public void blockUntilServerShutdown() {
                        modernServer.blockUntilShutdown();
                    }
                };
            } else {
                // ScalarToKey or KeyTKey
                final LegacyTcpPushServer<KeyValuePair<String, R>> modernServer = PushServers.infiniteStreamLegacyTcpNestedGroupedObservable(config, go, expiryTimeInSecs, keyEncoder, io.reactivex.mantis.network.push.HashFunctions.ketama());
                modernServer.start();
                // support legacy server interface
                this.server = new RemoteRxServer() {

                    @Override
                    public void start() {
                    }

                    @Override
                    public void startAndWait() {
                    }

                    @Override
                    public void shutdown() {
                        modernServer.shutdown();
                    }

                    @Override
                    public void blockUntilServerShutdown() {
                        modernServer.blockUntilShutdown();
                    }
                };
            }
        }
    } else if (stage instanceof ScalarToScalar || stage instanceof KeyToScalar || stage instanceof GroupToScalar) {
        if (runNewW2Wserver(jobName)) {
            logger.info("Modern server setup for name: " + name + " type: Scalarstage");
            Func1<R, byte[]> encoder = new Func1<R, byte[]>() {

                @Override
                public byte[] call(R t1) {
                    return stage.getOutputCodec().encode(t1);
                }
            };
            ServerConfig<R> config = new ServerConfig.Builder<R>().name(name).port(serverPort).metricsRegistry(MetricsRegistry.getInstance()).router(Routers.roundRobinLegacyTcpProtocol(name, encoder)).build();
            final LegacyTcpPushServer<R> modernServer = PushServers.infiniteStreamLegacyTcpNested(config, toServe);
            modernServer.start();
            // support legacy server interface
            this.server = new RemoteRxServer() {

                @Override
                public void start() {
                }

                @Override
                public void startAndWait() {
                }

                @Override
                public void shutdown() {
                    modernServer.shutdown();
                }

                @Override
                public void blockUntilServerShutdown() {
                    modernServer.blockUntilShutdown();
                }
            };
        } else {
            logger.info("Legacy server setup for name: " + name + " type: Scalarstage");
            RoundRobin slotting = new RoundRobin();
            serverBuilder.addObservable(new ServeNestedObservable.Builder().name(name).encoder(stage.getOutputCodec()).observable(toServe).slottingStrategy(slotting).build());
            MetricsRegistry.getInstance().registerAndGet(slotting.getMetrics());
            server = serverBuilder.port(serverPort).build();
            server.start();
        }
    } else {
        throw new RuntimeException("Unsupported stage type: " + stage);
    }
}

Example 2 with KeyToKey

use of io.mantisrx.runtime.KeyToKey in project mantis by Netflix.

the class StageExecutors method executeIntermediate.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static <T, R> void executeIntermediate(WorkerConsumer consumer, final StageConfig<T, R> stage, WorkerPublisher publisher, final Context context) {
    if (consumer == null) {
        throw new IllegalArgumentException("consumer cannot be null");
    }
    if (stage == null) {
        throw new IllegalArgumentException("stage cannot be null");
    }
    if (publisher == null) {
        throw new IllegalArgumentException("producer cannot be null");
    }
    Observable<?> toSink = null;
    if (stage instanceof ScalarToScalar) {
        ScalarToScalar scalarStage = (ScalarToScalar) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToScalarStage(scalarStage, source, context);
    } else if (stage instanceof ScalarToKey) {
        ScalarToKey scalarStage = (ScalarToKey) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToKeyStage(scalarStage, source, context);
    } else // NJ
    if (stage instanceof ScalarToGroup) {
        ScalarToGroup scalarStage = (ScalarToGroup) stage;
        Observable<Observable<T>> source = consumer.start(scalarStage);
        toSink = setupScalarToGroupStage(scalarStage, source, context);
    } else if (stage instanceof KeyToKey) {
        KeyToKey keyToKey = (KeyToKey) stage;
        Observable<Observable<GroupedObservable<String, T>>> source = consumer.start(keyToKey);
        toSink = setupKeyToKeyStage(keyToKey, source, context);
    } else if (stage instanceof GroupToGroup) {
        GroupToGroup groupToGroup = (GroupToGroup) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(groupToGroup);
        toSink = setupGroupToGroupStage(groupToGroup, source, context);
    } else if (stage instanceof KeyToScalar) {
        KeyToScalar scalarToKey = (KeyToScalar) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(scalarToKey);
        toSink = setupKeyToScalarStage(scalarToKey, source, context);
    } else if (stage instanceof GroupToScalar) {
        GroupToScalar groupToScalar = (GroupToScalar) stage;
        Observable<Observable<MantisGroup<String, T>>> source = consumer.start(groupToScalar);
        toSink = setupGroupToScalarStage(groupToScalar, source, context);
    }
    publisher.start(stage, toSink);
}