Examples with WorkerAssignments io.mantisrx.server.core.WorkerAssignments used on opensource projects

Example 1 with WorkerAssignments

use of io.mantisrx.server.core.WorkerAssignments in project mantis by Netflix.

the class JobRouteTest method testSchedulingInfo.

@Test(dependsOnMethods = { "testNamedJobInfoStream" })
public void testSchedulingInfo() throws InterruptedException {
    final CountDownLatch latch = new CountDownLatch(1);
    final String jobId = "sine-function-1";
    // final AtomicBoolean flag = new AtomicBoolean(false);
    Observable<JobSchedulingInfo> jobSchedulingInfoObservable = mantisClient.schedulingChanges(jobId);
    jobSchedulingInfoObservable.map(schedInfo -> {
        logger.info("schedInfo {}", schedInfo);
        try {
            assertEquals(jobId, schedInfo.getJobId());
            Map<Integer, WorkerAssignments> wa = schedInfo.getWorkerAssignments();
            assertEquals(2, wa.size());
            // 1 worker in stage 0
            assertEquals(1, wa.get(0).getHosts().size());
            assertEquals(0, wa.get(0).getHosts().get(1).getWorkerIndex());
            assertEquals(1, wa.get(0).getHosts().get(1).getWorkerNumber());
            assertEquals(MantisJobState.Started, wa.get(0).getHosts().get(1).getState());
            // 1 worker in stage 1
            assertEquals(1, wa.get(1).getHosts().size());
            assertEquals(0, wa.get(1).getHosts().get(2).getWorkerIndex());
            assertEquals(2, wa.get(1).getHosts().get(2).getWorkerNumber());
            assertEquals(MantisJobState.Started, wa.get(1).getHosts().get(2).getState());
        // if (flag.compareAndSet(false, true)) {
        // testJobResubmitWorker();
        // }
        } catch (Exception e) {
            logger.error("caught exception", e);
            org.testng.Assert.fail("testSchedulingInfo test failed with exception " + e.getMessage(), e);
        }
        latch.countDown();
        return schedInfo;
    }).take(1).doOnError(t -> logger.warn("onError", t)).doOnCompleted(() -> logger.info("onCompleted")).doAfterTerminate(() -> latch.countDown()).subscribe();
    latch.await();
}

Example 2 with WorkerAssignments

use of io.mantisrx.server.core.WorkerAssignments in project mantis by Netflix.

the class ConnectToNamedJob method main2.

public static void main2(final String[] args) {
    List<String> remArgs = Collections.emptyList();
    try {
        remArgs = Args.parse(ConnectToNamedJob.class, args);
    } catch (IllegalArgumentException e) {
        Args.usage(SubmitEphemeralJob.class);
        System.exit(1);
    }
    if (remArgs.isEmpty()) {
        System.err.println("Must provide JobId as argument");
        System.exit(1);
    }
    final String jobId = remArgs.get(0);
    Properties properties = new Properties();
    System.out.println("propfile=" + propFile);
    try (InputStream inputStream = new FileInputStream(propFile)) {
        properties.load(inputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }
    MasterClientWrapper clientWrapper = new MasterClientWrapper(properties);
    clientWrapper.getMasterClientApi().doOnNext(new Action1<MantisMasterClientApi>() {

        @Override
        public void call(MantisMasterClientApi clientApi) {
            logger.info("************** connecting to schedInfo for job " + jobId);
            clientApi.schedulingChanges(jobId).doOnNext(new Action1<JobSchedulingInfo>() {

                @Override
                public void call(JobSchedulingInfo schedulingInfo) {
                    final WorkerAssignments workerAssignments = schedulingInfo.getWorkerAssignments().get(1);
                    for (Map.Entry<Integer, WorkerHost> entry : workerAssignments.getHosts().entrySet()) {
                        System.out.println("Worker " + entry.getKey() + ": state=" + entry.getValue().getState() + ", host=" + entry.getValue().getHost() + ", port=" + entry.getValue().getPort());
                    }
                }
            }).subscribe();
            ;
        }
    }).subscribe();
    // .subscribe();
    try {
        Thread.sleep(10000000);
    } catch (InterruptedException ie) {
    }
}

Example 3 with WorkerAssignments

use of io.mantisrx.server.core.WorkerAssignments in project mantis by Netflix.

the class StageWorkersCount method getWorkerCounts.

Observable<Integer> getWorkerCounts(final int stageNumber) {
    final AtomicInteger workerCount = new AtomicInteger(0);
    final AtomicBoolean gotCompletion = new AtomicBoolean(false);
    return mantisClient.getClientWrapper().getMasterClientApi().flatMap(new Func1<MantisMasterClientApi, Observable<Integer>>() {

        @Override
        public Observable<Integer> call(MantisMasterClientApi mantisMasterClientApi) {
            return mantisMasterClientApi.schedulingChanges(jobId).map(new Func1<JobSchedulingInfo, Integer>() {

                @Override
                public Integer call(JobSchedulingInfo jobSchedulingInfo) {
                    final WorkerAssignments assignments = jobSchedulingInfo.getWorkerAssignments().get(stageNumber);
                    if (assignments == null)
                        return -1;
                    else
                        return assignments.getNumWorkers();
                }
            }).filter(new Func1<Integer, Boolean>() {

                @Override
                public Boolean call(Integer newCount) {
                    if (newCount == workerCount.get())
                        return false;
                    workerCount.set(newCount);
                    return true;
                }
            }).doOnCompleted(new Action0() {

                @Override
                public void call() {
                    gotCompletion.set(true);
                }
            });
        }
    }).takeWhile(new Func1<Integer, Boolean>() {

        @Override
        public Boolean call(Integer integer) {
            return !gotCompletion.get();
        }
    });
}

Example 4 with WorkerAssignments

use of io.mantisrx.server.core.WorkerAssignments in project mantis by Netflix.

the class MasterClientWrapper method getAllNonJobMasterEndpoints.

private List<Endpoint> getAllNonJobMasterEndpoints(final String jobId, final Map<Integer, WorkerAssignments> workerAssignments) {
    List<Endpoint> endpoints = new ArrayList<>();
    int totalWorkers = 0;
    for (Map.Entry<Integer, WorkerAssignments> workerAssignment : workerAssignments.entrySet()) {
        final Integer stageNum = workerAssignment.getKey();
        // skip workers for stage 0
        if (stageNum == 0) {
            continue;
        }
        final WorkerAssignments assignments = workerAssignment.getValue();
        logger.info("job {} Creating endpoints conx from {} worker assignments for stage {}", jobId, assignments.getHosts().size(), stageNum);
        if (logger.isDebugEnabled()) {
            logger.debug("stage {} hosts: {}", stageNum, assignments.getHosts());
        }
        totalWorkers += assignments.getNumWorkers();
        for (WorkerHost host : assignments.getHosts().values()) {
            final int workerIndex = host.getWorkerIndex();
            if (host.getState() == MantisJobState.Started) {
                logger.info("job " + jobId + ": creating new endpoint for worker number=" + host.getWorkerNumber() + ", index=" + host.getWorkerIndex() + ", host:port=" + host.getHost() + ":" + host.getPort().get(0));
                Endpoint ep = new WorkerEndpoint(getWrappedHost(host.getHost(), host.getWorkerNumber()), host.getPort().get(0), stageNum, host.getMetricsPort(), host.getWorkerIndex(), host.getWorkerNumber(), // completed callback
                new Action0() {

                    @Override
                    public void call() {
                        logger.info("job " + jobId + " WorkerIndex " + workerIndex + " completed");
                    }
                }, // error callback
                new Action1<Throwable>() {

                    @Override
                    public void call(Throwable t1) {
                        logger.info("job " + jobId + " WorkerIndex " + workerIndex + " failed");
                    }
                });
                endpoints.add(ep);
            }
        }
    }
    numWorkersSubject.onNext(new JobNumWorkers(jobId, totalWorkers));
    return endpoints;
}

Example 5 with WorkerAssignments

use of io.mantisrx.server.core.WorkerAssignments in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method connectToObservableAtPreviousStages.

@SuppressWarnings({ "rawtypes" })
private WorkerConsumer connectToObservableAtPreviousStages(Observable<JobSchedulingInfo> selfSchedulingInfo, final String jobId, final int previousStageNum, int numInstanceAtPreviousStage, final StageConfig previousStage, final AtomicBoolean acceptSchedulingChanges, final Observer<Status> jobStatus, final int stageNumToExecute, final int workerIndex, final int workerNumber) {
    logger.info("Watching for scheduling changes");
    // Observable<List<Endpoint>> schedulingUpdates = mantisMasterApi.schedulingChanges(jobId)
    Observable<List<Endpoint>> schedulingUpdates = selfSchedulingInfo.flatMap((Func1<JobSchedulingInfo, Observable<WorkerAssignments>>) schedulingChange -> {
        Map<Integer, WorkerAssignments> assignments = schedulingChange.getWorkerAssignments();
        if (assignments != null && !assignments.isEmpty()) {
            return Observable.from(assignments.values());
        } else {
            return Observable.empty();
        }
    }).filter(assignments -> (assignments.getStage() == previousStageNum) && acceptSchedulingChanges.get()).map(assignments -> {
        List<Endpoint> endpoints = new LinkedList<>();
        for (WorkerHost host : assignments.getHosts().values()) {
            if (host.getState() == MantisJobState.Started) {
                logger.info("Received scheduling update from master, connect request for host: " + host.getHost() + " port: " + host.getPort() + " state: " + host.getState() + " adding: " + connectionsPerEndpoint + " connections to host");
                for (int i = 1; i <= connectionsPerEndpoint; i++) {
                    final String endpointId = "stage_" + stageNumToExecute + "_index_" + Integer.toString(workerIndex) + "_partition_" + i;
                    logger.info("Adding endpoint to endpoint injector to be considered for add, with id: " + endpointId);
                    endpoints.add(new Endpoint(host.getHost(), host.getPort().get(0), endpointId));
                }
            }
        }
        return endpoints;
    }).filter(t1 -> (t1.size() > 0));
    String name = jobId + "_" + previousStageNum;
    return new WorkerConsumerRemoteObservable(name, new ToDeltaEndpointInjector(schedulingUpdates));
}