Examples with EndpointChange io.reactivex.mantis.remote.observable.EndpointChange used on opensource projects

Example 6 with EndpointChange

use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.

the class Reconciliator method deltas.

private Observable<EndpointChange> deltas() {
    final Map<String, Endpoint> sideEffectState = new HashMap<String, Endpoint>();
    final PublishSubject<Integer> stopReconciliator = PublishSubject.create();
    return Observable.merge(reconciledChanges.takeUntil(stopReconciliator).doOnCompleted(() -> {
        logger.info("onComplete triggered for reconciledChanges");
    }).doOnError(e -> logger.error("caught exception for reconciledChanges {}", e.getMessage(), e)), injector.deltas().doOnCompleted(new Action0() {

        @Override
        public void call() {
            // injector has completed recieving updates, complete reconciliator
            // observable
            logger.info("Stopping reconciliator, injector completed.");
            stopReconciliator.onNext(1);
            stopReconciliation();
        }
    }).doOnError(e -> logger.error("caught exception for injector deltas {}", e.getMessage(), e)).doOnNext(new Action1<EndpointChange>() {

        @Override
        public void call(EndpointChange newEndpointChange) {
            String id = Endpoint.uniqueHost(newEndpointChange.getEndpoint().getHost(), newEndpointChange.getEndpoint().getPort(), newEndpointChange.getEndpoint().getSlotId());
            if (sideEffectState.containsKey(id)) {
                if (newEndpointChange.getType() == Type.complete) {
                    // remove from expecected set
                    expectedSetSize.decrement();
                    sideEffectState.remove(id);
                    currentExpectedSet.onNext(new HashSet<Endpoint>(sideEffectState.values()));
                }
            } else {
                if (newEndpointChange.getType() == Type.add) {
                    expectedSetSize.increment();
                    sideEffectState.put(id, new Endpoint(newEndpointChange.getEndpoint().getHost(), newEndpointChange.getEndpoint().getPort(), newEndpointChange.getEndpoint().getSlotId()));
                    currentExpectedSet.onNext(new HashSet<Endpoint>(sideEffectState.values()));
                }
            }
        }
    })).doOnError(t -> logger.error("caught error processing reconciliator deltas {}", t.getMessage(), t)).doOnSubscribe(new Action0() {

        @Override
        public void call() {
            logger.info("Subscribed to deltas for {}, clearing active connection set", name);
            connectionSet.resetActiveConnections();
            startReconciliation();
        }
    }).doOnUnsubscribe(new Action0() {

        @Override
        public void call() {
            logger.info("Unsubscribed from deltas for {}", name);
        }
    });
}

Example 7 with EndpointChange

use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.

the class MasterClientWrapper method getSinkLocations.

public Observable<EndpointChange> getSinkLocations(final String jobId, final int sinkStage, final int forPartition, final int totalPartitions) {
    final ConditionalRetry schedInfoRetry = new ConditionalRetry(masterConnectRetryCounter, "SchedInfoRetry", 10);
    Observable<List<Endpoint>> schedulingUpdates = getMasterClientApi().take(1).flatMap((MantisMasterClientApi mantisMasterClientApi) -> {
        return mantisMasterClientApi.schedulingChanges(jobId).doOnError((Throwable throwable) -> {
            logger.warn(throwable.getMessage());
        }).retryWhen(schedInfoRetry.getRetryLogic()).map((JobSchedulingInfo jobSchedulingInfo) -> {
            logger.info("Got scheduling info for {}", jobId);
            if (logger.isDebugEnabled()) {
                logger.debug("Worker Assignments {}", jobSchedulingInfo.getWorkerAssignments().get(sinkStage));
            }
            return jobSchedulingInfo.getWorkerAssignments().get(sinkStage);
        }).map((WorkerAssignments workerAssignments) -> {
            List<Endpoint> endpoints = new ArrayList<>();
            if (workerAssignments != null) {
                logger.info("job " + jobId + " Creating endpoints conx from " + workerAssignments.getHosts().size() + " worker assignments");
                for (WorkerHost host : workerAssignments.getHosts().values()) {
                    final int workerIndex = host.getWorkerIndex();
                    final int totalFromPartitions = workerAssignments.getNumWorkers();
                    numSinkWorkersSubject.onNext(new JobSinkNumWorkers(jobId, totalFromPartitions));
                    if (usePartition(workerIndex, totalFromPartitions, forPartition, totalPartitions)) {
                        // logger.info("Using partition " + workerIndex);
                        if (host.getState() == MantisJobState.Started) {
                            Endpoint ep = new Endpoint(getWrappedHost(host.getHost(), host.getWorkerNumber()), host.getPort().get(0), // completed callback
                            () -> logger.info("job " + jobId + " WorkerIndex " + workerIndex + " completed"), // error callback
                            t1 -> logger.info("job " + jobId + " WorkerIndex " + workerIndex + " failed"));
                            endpoints.add(ep);
                        }
                    }
                }
            } else {
                logger.info("job " + jobId + " Has no active workers!");
            }
            return endpoints;
        }).doOnError((Throwable throwable) -> {
            logger.error(throwable.getMessage(), throwable);
        });
    });
    return (new ToDeltaEndpointInjector(schedulingUpdates)).deltas();
}

Example 8 with EndpointChange

use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.

the class MasterClientWrapperTest method getSchedulingInfoTest.

// @Test
public void getSchedulingInfoTest() {
    String jobname = "GroupByIP";
    CountDownLatch cdLatch = new CountDownLatch(3);
    Observable<String> jobidO = clientWrapper.getNamedJobsIds(jobname).take(1).cache().subscribeOn(Schedulers.io());
    Observable<MantisMasterClientApi> mmciO = clientWrapper.getMasterClientApi().take(1).cache().subscribeOn(Schedulers.io());
    Observable<EndpointChange> epO = jobidO.map((jId) -> clientWrapper.getSinkLocations(jId, sinkStageNumber, 0, 0)).flatMap(e -> e).take(3).doOnNext((ep) -> System.out.println("Ep change: " + ep)).doOnNext((ep) -> cdLatch.countDown());
    Observable<Boolean> deleteWorkerO = jobidO.zipWith(mmciO, (String jId, MantisMasterClientApi mmci) -> {
        System.out.println("Job id is " + jId);
        return mmci.schedulingChanges(jId).map(jsi -> {
            Map<Integer, WorkerAssignments> workerAssignments = jsi.getWorkerAssignments();
            System.out.println("WorkerAssignments -> " + workerAssignments);
            WorkerAssignments workerAssignmentsForSink = workerAssignments.get(sinkStageNumber);
            System.out.println("WorkerAssignmentsForSink -> " + workerAssignmentsForSink);
            Map<Integer, WorkerHost> hostsForSink = workerAssignmentsForSink.getHosts();
            System.out.println("Host map -> " + hostsForSink);
            assertTrue(!hostsForSink.isEmpty());
            Iterator<Entry<Integer, WorkerHost>> it = hostsForSink.entrySet().iterator();
            while (it.hasNext()) {
                Entry<Integer, WorkerHost> e = it.next();
                return e.getValue().getWorkerNumber();
            }
            return -1;
        }).take(1).map((Integer workerNo) -> {
            System.out.println("Worker no is -> " + workerNo);
            return mmci.resubmitJobWorker(jId, "tester", workerNo, "testing");
        }).flatMap(b -> b);
    }).flatMap(b -> b).doOnNext((result) -> {
        assertTrue(result);
        cdLatch.countDown();
    });
    epO.subscribeOn(Schedulers.io()).subscribe((ep) -> System.out.println(ep), (t) -> t.printStackTrace(), () -> System.out.println("ep change completed"));
    deleteWorkerO.toBlocking().subscribe((n) -> System.out.println(n), (t) -> t.printStackTrace(), () -> System.out.println("worker deletion completed"));
    try {
        cdLatch.await(10, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        fail();
    }
}

Example 9 with EndpointChange

use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.

the class MetricsClientImplTest method testMetricConnections.

@Test
public void testMetricConnections() throws InterruptedException, UnsupportedEncodingException, JsonProcessingException {
    final String jobId = "test-job-1";
    final String testResUsageMetricData = generateMetricJson(MetricStringConstants.RESOURCE_USAGE_METRIC_GROUP);
    final String testDropDataMetricData = generateMetricJson(MetricStringConstants.DATA_DROP_METRIC_GROUP);
    final int metricsPort = TestSseServerFactory.newServerWithInitialData(testResUsageMetricData);
    final AtomicInteger i = new AtomicInteger(0);
    final Observable<EndpointChange> workerMetricLocationStream = Observable.interval(1, TimeUnit.SECONDS, Schedulers.io()).map(new Func1<Long, EndpointChange>() {

        @Override
        public EndpointChange call(Long aLong) {
            logger.info("emitting endpointChange");
            if (i.getAndIncrement() % 10 == 0) {
                return new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", 31002));
            } else {
                return new EndpointChange(EndpointChange.Type.add, new WorkerEndpoint("localhost", 31002, 1, metricsPort, 0, 1));
            }
        }
    });
    MetricsClientImpl<MantisServerSentEvent> metricsClient = new MetricsClientImpl<>(jobId, new SseWorkerConnectionFunction(true, new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            logger.error("Metric connection error: " + throwable.getMessage());
            try {
                Thread.sleep(500);
            } catch (InterruptedException ie) {
                logger.error("Interrupted waiting for retrying connection");
            }
        }
    }, new SinkParameters.Builder().withParameter("name", MetricStringConstants.RESOURCE_USAGE_METRIC_GROUP).build()), new JobWorkerMetricsLocator() {

        @Override
        public Observable<EndpointChange> locateWorkerMetricsForJob(String jobId) {
            return workerMetricLocationStream;
        }
    }, Observable.just(1), new Observer<WorkerConnectionsStatus>() {

        @Override
        public void onCompleted() {
            logger.info("got onCompleted in WorkerConnStatus obs");
        }

        @Override
        public void onError(Throwable e) {
            logger.info("got onError in WorkerConnStatus obs");
        }

        @Override
        public void onNext(WorkerConnectionsStatus workerConnectionsStatus) {
            logger.info("got WorkerConnStatus {}", workerConnectionsStatus);
        }
    }, 60);
    final CountDownLatch latch = new CountDownLatch(1);
    final Observable<Observable<MantisServerSentEvent>> results = metricsClient.getResults();
    Observable.merge(results).doOnNext(new Action1<MantisServerSentEvent>() {

        @Override
        public void call(MantisServerSentEvent event) {
            logger.info("got event {}", event.getEventAsString());
            assertEquals(testResUsageMetricData, event.getEventAsString());
            latch.countDown();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            logger.error("got error {}", throwable.getMessage(), throwable);
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            logger.info("onComplete");
        }
    }).subscribe();
    latch.await(30, TimeUnit.SECONDS);
    TestSseServerFactory.stopAllRunning();
}