Examples with RemoteRxServer io.reactivex.mantis.remote.observable.RemoteRxServer used on opensource projects

Example 1 with RemoteRxServer

use of io.reactivex.mantis.remote.observable.RemoteRxServer in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method executeStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void executeStage(final ExecutionDetails setup) {
    ExecuteStageRequest executionRequest = setup.getExecuteStageRequest().getRequest();
    // Initialize the schedulingInfo observable for current job and mark it shareable to be reused by anyone interested in this data.
    // Observable<JobSchedulingInfo> selfSchedulingInfo = mantisMasterApi.schedulingChanges(executionRequest.getJobId()).switchMap((e) -> Observable.just(e).repeatWhen(x -> x.delay(5 , TimeUnit.SECONDS))).subscribeOn(Schedulers.io()).share();
    Observable<JobSchedulingInfo> selfSchedulingInfo = mantisMasterApi.schedulingChanges(executionRequest.getJobId()).subscribeOn(Schedulers.io()).share();
    WorkerInfo workerInfo = generateWorkerInfo(executionRequest.getJobName(), executionRequest.getJobId(), executionRequest.getStage(), executionRequest.getWorkerIndex(), executionRequest.getWorkerNumber(), executionRequest.getDurationType(), "host", executionRequest.getWorkerPorts());
    final Observable<Integer> sourceStageTotalWorkersObs = createSourceStageTotalWorkersObservable(selfSchedulingInfo);
    RunningWorker.Builder rwBuilder = new RunningWorker.Builder().job(setup.getMantisJob()).schedulingInfo(executionRequest.getSchedulingInfo()).stageTotalWorkersObservable(sourceStageTotalWorkersObs).jobName(executionRequest.getJobName()).stageNum(executionRequest.getStage()).workerIndex(executionRequest.getWorkerIndex()).workerNum(executionRequest.getWorkerNumber()).totalStages(executionRequest.getTotalNumStages()).metricsPort(executionRequest.getMetricsPort()).ports(executionRequest.getPorts().iterator()).jobStatusObserver(setup.getStatus()).requestSubject(setup.getExecuteStageRequest().getRequestSubject()).workerInfo(workerInfo).vmTaskStatusObservable(vmTaskStatusObserver).hasJobMaster(executionRequest.getHasJobMaster()).jobId(executionRequest.getJobId());
    if (executionRequest.getStage() == 0) {
        rwBuilder = rwBuilder.stage(new JobMasterStageConfig("jobmasterconfig"));
    } else {
        rwBuilder = rwBuilder.stage((StageConfig) setup.getMantisJob().getStages().get(executionRequest.getStage() - 1));
    }
    final RunningWorker rw = rwBuilder.build();
    AtomicReference<SubscriptionStateHandler> subscriptionStateHandlerRef = new AtomicReference<>();
    if (rw.getStageNum() == rw.getTotalStagesNet()) {
        // set up subscription state handler only for sink (last) stage
        subscriptionStateHandlerRef.set(setupSubscriptionStateHandler(setup.getExecuteStageRequest().getRequest().getJobId(), mantisMasterApi, setup.getExecuteStageRequest().getRequest().getSubscriptionTimeoutSecs(), setup.getExecuteStageRequest().getRequest().getMinRuntimeSecs()));
    }
    logger.info("Running worker info: " + rw);
    rw.signalStartedInitiated();
    try {
        logger.info(">>>>>>>>>>>>>>>>Calling lifecycle.startup()");
        Lifecycle lifecycle = rw.getJob().getLifecycle();
        lifecycle.startup();
        ServiceLocator serviceLocator = lifecycle.getServiceLocator();
        if (lookupSpectatorRegistry) {
            try {
                final Registry spectatorRegistry = serviceLocator.service(Registry.class);
                SpectatorRegistryFactory.setRegistry(spectatorRegistry);
            } catch (Throwable t) {
                logger.error("failed to init spectator registry using service locator, falling back to {}", SpectatorRegistryFactory.getRegistry().getClass().getCanonicalName());
            }
        }
        // create job context
        Parameters parameters = ParameterUtils.createContextParameters(rw.getJob().getParameterDefinitions(), setup.getParameters());
        final Context context = generateContext(parameters, serviceLocator, workerInfo, MetricsRegistry.getInstance(), () -> {
            rw.signalCompleted();
            // wait for completion signal to go to the master and us getting killed. Upon timeout, exit.
            try {
                Thread.sleep(60000);
            } catch (InterruptedException ie) {
                logger.warn("Unexpected exception sleeping: " + ie.getMessage());
            }
            System.exit(0);
        }, createWorkerMapObservable(selfSchedulingInfo, executionRequest.getJobName(), executionRequest.getJobId(), executionRequest.getDurationType()));
        // context.setPrevStageCompletedObservable(createPrevStageCompletedObservable(selfSchedulingInfo, rw.getJobId(), rw.getStageNum()));
        rw.setContext(context);
        // setup heartbeats
        heartbeatRef.set(new Heartbeat(rw.getJobId(), rw.getStageNum(), rw.getWorkerIndex(), rw.getWorkerNum()));
        final double networkMbps = executionRequest.getSchedulingInfo().forStage(rw.getStageNum()).getMachineDefinition().getNetworkMbps();
        startSendingHeartbeats(rw.getJobStatus(), new WorkerId(executionRequest.getJobId(), executionRequest.getWorkerIndex(), executionRequest.getWorkerNumber()).getId(), networkMbps);
        // execute stage
        if (rw.getStageNum() == 0) {
            logger.info("JobId: " + rw.getJobId() + ", executing Job Master");
            final AutoScaleMetricsConfig autoScaleMetricsConfig = new AutoScaleMetricsConfig();
            // Temporary workaround to enable auto-scaling by custom metric in Job Master. This will be revisited to get the entire autoscaling config
            // for a job as a System parameter in the JobMaster
            final String autoScaleMetricString = (String) parameters.get(JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM, "");
            if (!Strings.isNullOrEmpty(autoScaleMetricString)) {
                final List<String> tokens = Splitter.on("::").omitEmptyStrings().trimResults().splitToList(autoScaleMetricString);
                if (tokens.size() == 3) {
                    final String metricGroup = tokens.get(0);
                    final String metricName = tokens.get(1);
                    final String algo = tokens.get(2);
                    try {
                        final AutoScaleMetricsConfig.AggregationAlgo aggregationAlgo = AutoScaleMetricsConfig.AggregationAlgo.valueOf(algo);
                        logger.info("registered UserDefined auto scale metric {}:{} algo {}", metricGroup, metricName, aggregationAlgo);
                        autoScaleMetricsConfig.addUserDefinedMetric(metricGroup, metricName, aggregationAlgo);
                    } catch (IllegalArgumentException e) {
                        final String errorMsg = String.format("ERROR: Invalid algorithm value %s for param %s (algo should be one of %s)", autoScaleMetricsConfig, JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM, Arrays.stream(AutoScaleMetricsConfig.AggregationAlgo.values()).map(a -> a.name()).collect(Collectors.toList()));
                        logger.error(errorMsg);
                        throw new RuntimeException(errorMsg);
                    }
                } else {
                    final String errorMsg = String.format("ERROR: Invalid value %s for param %s", autoScaleMetricString, JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM);
                    logger.error(errorMsg);
                    throw new RuntimeException(errorMsg);
                }
            } else {
                logger.info("param {} is null or empty", JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM);
            }
            final JobMasterService jobMasterService = new JobMasterService(rw.getJobId(), rw.getSchedulingInfo(), workerMetricsClient, autoScaleMetricsConfig, mantisMasterApi, rw.getContext(), rw.getOnCompleteCallback(), rw.getOnErrorCallback(), rw.getOnTerminateCallback());
            jobMasterService.start();
            signalStarted(rw, subscriptionStateHandlerRef);
            // block until worker terminates
            rw.waitUntilTerminate();
        } else if (rw.getStageNum() == 1 && rw.getTotalStagesNet() == 1) {
            logger.info("JobId: " + rw.getJobId() + ", single stage job, executing entire job");
            // single stage, execute entire job on this machine
            PortSelector portSelector = new PortSelector() {

                @Override
                public int acquirePort() {
                    return rw.getPorts().next();
                }
            };
            RxMetrics rxMetrics = new RxMetrics();
            StageExecutors.executeSingleStageJob(rw.getJob().getSource(), rw.getStage(), rw.getJob().getSink(), portSelector, rxMetrics, rw.getContext(), rw.getOnTerminateCallback(), rw.getWorkerIndex(), rw.getSourceStageTotalWorkersObservable(), onSinkSubscribe, onSinkUnsubscribe, rw.getOnCompleteCallback(), rw.getOnErrorCallback());
            signalStarted(rw, subscriptionStateHandlerRef);
            // block until worker terminates
            rw.waitUntilTerminate();
        } else {
            logger.info("JobId: " + rw.getJobId() + ", executing a multi-stage job, stage: " + rw.getStageNum());
            if (rw.getStageNum() == 1) {
                // execute source stage
                String remoteObservableName = rw.getJobId() + "_" + rw.getStageNum();
                StageSchedulingInfo currentStageSchedulingInfo = rw.getSchedulingInfo().forStage(1);
                WorkerPublisherRemoteObservable publisher = new WorkerPublisherRemoteObservable<>(rw.getPorts().next(), remoteObservableName, numWorkersAtStage(selfSchedulingInfo, rw.getJobId(), rw.getStageNum() + 1), rw.getJobName());
                StageExecutors.executeSource(rw.getWorkerIndex(), rw.getJob().getSource(), rw.getStage(), publisher, rw.getContext(), rw.getSourceStageTotalWorkersObservable());
                logger.info("JobId: " + rw.getJobId() + " stage: " + rw.getStageNum() + ", serving remote observable for source with name: " + remoteObservableName);
                RemoteRxServer server = publisher.getServer();
                RxMetrics rxMetrics = server.getMetrics();
                MetricsRegistry.getInstance().registerAndGet(rxMetrics.getCountersAndGauges());
                signalStarted(rw, subscriptionStateHandlerRef);
                logger.info("JobId: " + rw.getJobId() + " stage: " + rw.getStageNum() + ", blocking until source observable completes");
                server.blockUntilServerShutdown();
            } else {
                // execute intermediate stage or last stage plus sink
                executeNonSourceStage(selfSchedulingInfo, rw, subscriptionStateHandlerRef);
            }
        }
        logger.info("Calling lifecycle.shutdown()");
        lifecycle.shutdown();
    } catch (Throwable t) {
        rw.signalFailed(t);
        shutdownStage();
    }
}

Example 2 with RemoteRxServer

use of io.reactivex.mantis.remote.observable.RemoteRxServer in project mantis by Netflix.

the class StageExecutorsGroupByTest method testExecuteIntermediatStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
    // Note, this test has a timing issue, client starts
    // sending data before server is ready, resulting
    // in a RST (connection reset by peer)
    TestGroupByJob provider = new TestGroupByJob();
    Job<Pair> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int publishPort = portSelector.acquirePort();
    final int consumerPort = portSelector.acquirePort();
    Observable<Observable<GroupedObservable<String, Integer>>> go = Observable.just(Observable.range(0, 10).groupBy(new Func1<Integer, String>() {

        @Override
        public String call(Integer t1) {
            if ((t1 % 2) == 0) {
                return "even";
            } else {
                return "odd";
            }
        }
    }));
    // mimic previous stage with a server
    ServeGroupedObservable<String, Integer> config = new ServeGroupedObservable.Builder<String, Integer>().keyEncoder(Codecs.string()).valueEncoder(Codecs.integer()).observable(go).build();
    RemoteRxServer server = new RemoteRxServer.Builder().addObservable(config).port(consumerPort).build();
    server.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "0")));
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
    // execute source
    StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
    ConnectToGroupedObservable<String, Integer> connectConfig = new ConnectToGroupedObservable.Builder<String, Integer>().host("localhost").port(publishPort).keyDecoder(Codecs.string()).valueDecoder(Codecs.integer()).build();
    Iterator<GroupedObservable<String, Integer>> iter = RemoteObservable.connect(connectConfig).getObservable().toBlocking().getIterator();
    // verify numbers are grouped by even/odd
    // even is first due to zero
    GroupedObservable<String, Integer> even = iter.next();
    Assert.assertEquals("even", even.getKey());
    Iterator<Integer> evenIter = even.toBlocking().getIterator();
    Assert.assertEquals(0, evenIter.next().intValue());
    Assert.assertEquals(4, evenIter.next().intValue());
    Assert.assertEquals(16, evenIter.next().intValue());
    Assert.assertEquals(36, evenIter.next().intValue());
    GroupedObservable<String, Integer> odd = iter.next();
    Assert.assertEquals("odd", odd.getKey());
    Iterator<Integer> oddIter = odd.toBlocking().getIterator();
    Assert.assertEquals(1, oddIter.next().intValue());
    Assert.assertEquals(9, oddIter.next().intValue());
    Assert.assertEquals(25, oddIter.next().intValue());
    Assert.assertEquals(49, oddIter.next().intValue());
    // should only have two groups
    Assert.assertEquals(false, iter.hasNext());
}

Example 3 with RemoteRxServer

use of io.reactivex.mantis.remote.observable.RemoteRxServer in project mantis by Netflix.

the class StageExecutorsTest method testExecuteIntermediatStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
    TestJob provider = new TestJob();
    Job<Integer> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int publishPort = portSelector.acquirePort();
    final int consumerPort = portSelector.acquirePort();
    // mimic previous stage with a server
    RemoteRxServer server1 = RemoteObservable.serve(consumerPort, Observable.range(0, 10), Codecs.integer());
    server1.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
    // execute intermediate, flatten results
    StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
    Iterator<Integer> iter = RemoteObservable.connect(new ConnectToObservable.Builder<Integer>().host("localhost").slotId("0").port(publishPort).decoder(Codecs.integer()).build()).getObservable().toBlocking().getIterator();
    // verify numbers are even
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(2, iter.next().intValue());
    Assert.assertEquals(4, iter.next().intValue());
}

Example 4 with RemoteRxServer

use of io.reactivex.mantis.remote.observable.RemoteRxServer in project mantis by Netflix.

the class StageExecutorsTest method testExecuteSink.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSink() throws InterruptedException {
    TestJob provider = new TestJob();
    Job<Integer> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int consumerPort = portSelector.acquirePort();
    // mimic previous stage with a server
    RemoteRxServer server1 = RemoteObservable.serve(consumerPort, Observable.range(0, 10), Codecs.integer());
    server1.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    Action0 noOpAction = new Action0() {

        @Override
        public void call() {
        }
    };
    Action1<Throwable> noOpError = new Action1<Throwable>() {

        @Override
        public void call(Throwable t) {
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    // execute source
    StageExecutors.executeSink(consumer, stages.get(1), job.getSink(), new TestPortSelector(), new RxMetrics(), new Context(), noOpAction, null, null, noOpAction, noOpError);
    Iterator<Integer> iter = provider.getItemsWritten().iterator();
    // verify numbers are even
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(2, iter.next().intValue());
    Assert.assertEquals(4, iter.next().intValue());
}

Example 5 with RemoteRxServer

use of io.reactivex.mantis.remote.observable.RemoteRxServer in project mantis by Netflix.

the class WorkerPublisherRemoteObservable method start.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void start(final StageConfig<T, R> stage, Observable<Observable<R>> toServe) {
    RemoteRxServer.Builder serverBuilder = new RemoteRxServer.Builder();
    if (stage instanceof KeyToKey || stage instanceof ScalarToKey || stage instanceof ScalarToGroup || stage instanceof GroupToGroup) {
        if (runNewW2WserverGroups(jobName)) {
            logger.info("Modern server setup for name: " + name + " type: Keyedstage");
            long expiryTimeInSecs = Long.MAX_VALUE;
            if (stage instanceof KeyToKey) {
                expiryTimeInSecs = ((KeyToKey) stage).getKeyExpireTimeSeconds();
            } else if (stage instanceof ScalarToKey) {
                expiryTimeInSecs = ((ScalarToKey) stage).getKeyExpireTimeSeconds();
            }
            Func1<R, byte[]> valueEncoder = new Func1<R, byte[]>() {

                @Override
                public byte[] call(R t1) {
                    return stage.getOutputCodec().encode(t1);
                }
            };
            Func1<String, byte[]> keyEncoder = new Func1<String, byte[]>() {

                @Override
                public byte[] call(String t1) {
                    return Codecs.string().encode(t1);
                }
            };
            ServerConfig<KeyValuePair<String, R>> config = new ServerConfig.Builder<KeyValuePair<String, R>>().name(name).port(serverPort).metricsRegistry(MetricsRegistry.getInstance()).numQueueConsumers(numConsumerThreads()).maxChunkSize(maxChunkSize()).maxChunkTimeMSec(maxChunkTimeMSec()).bufferCapacity(bufferCapacity()).useSpscQueue(useSpsc()).router(Routers.consistentHashingLegacyTcpProtocol(jobName, keyEncoder, valueEncoder)).build();
            // remove type
            Observable go = toServe;
            if (stage instanceof ScalarToGroup || stage instanceof GroupToGroup) {
                final LegacyTcpPushServer<KeyValuePair<String, R>> modernServer = PushServers.infiniteStreamLegacyTcpNestedMantisGroup(config, go, expiryTimeInSecs, keyEncoder, io.reactivex.mantis.network.push.HashFunctions.ketama());
                modernServer.start();
                // support legacy server interface
                this.server = new RemoteRxServer() {

                    @Override
                    public void start() {
                    }

                    @Override
                    public void startAndWait() {
                    }

                    @Override
                    public void shutdown() {
                        modernServer.shutdown();
                    }

                    @Override
                    public void blockUntilServerShutdown() {
                        modernServer.blockUntilShutdown();
                    }
                };
            } else {
                // ScalarToKey or KeyTKey
                final LegacyTcpPushServer<KeyValuePair<String, R>> modernServer = PushServers.infiniteStreamLegacyTcpNestedGroupedObservable(config, go, expiryTimeInSecs, keyEncoder, io.reactivex.mantis.network.push.HashFunctions.ketama());
                modernServer.start();
                // support legacy server interface
                this.server = new RemoteRxServer() {

                    @Override
                    public void start() {
                    }

                    @Override
                    public void startAndWait() {
                    }

                    @Override
                    public void shutdown() {
                        modernServer.shutdown();
                    }

                    @Override
                    public void blockUntilServerShutdown() {
                        modernServer.blockUntilShutdown();
                    }
                };
            }
        }
    } else if (stage instanceof ScalarToScalar || stage instanceof KeyToScalar || stage instanceof GroupToScalar) {
        if (runNewW2Wserver(jobName)) {
            logger.info("Modern server setup for name: " + name + " type: Scalarstage");
            Func1<R, byte[]> encoder = new Func1<R, byte[]>() {

                @Override
                public byte[] call(R t1) {
                    return stage.getOutputCodec().encode(t1);
                }
            };
            ServerConfig<R> config = new ServerConfig.Builder<R>().name(name).port(serverPort).metricsRegistry(MetricsRegistry.getInstance()).router(Routers.roundRobinLegacyTcpProtocol(name, encoder)).build();
            final LegacyTcpPushServer<R> modernServer = PushServers.infiniteStreamLegacyTcpNested(config, toServe);
            modernServer.start();
            // support legacy server interface
            this.server = new RemoteRxServer() {

                @Override
                public void start() {
                }

                @Override
                public void startAndWait() {
                }

                @Override
                public void shutdown() {
                    modernServer.shutdown();
                }

                @Override
                public void blockUntilServerShutdown() {
                    modernServer.blockUntilShutdown();
                }
            };
        } else {
            logger.info("Legacy server setup for name: " + name + " type: Scalarstage");
            RoundRobin slotting = new RoundRobin();
            serverBuilder.addObservable(new ServeNestedObservable.Builder().name(name).encoder(stage.getOutputCodec()).observable(toServe).slottingStrategy(slotting).build());
            MetricsRegistry.getInstance().registerAndGet(slotting.getMetrics());
            server = serverBuilder.port(serverPort).build();
            server.start();
        }
    } else {
        throw new RuntimeException("Unsupported stage type: " + stage);
    }
}