Examples with WorkerMap io.mantisrx.runtime.WorkerMap used on opensource projects

Example 1 with WorkerMap

use of io.mantisrx.runtime.WorkerMap in project mantis by Netflix.

the class LocalJobExecutorNetworked method execute.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void execute(Job job, SchedulingInfo schedulingInfo, Parameter... parameters) throws IllegalMantisJobException {
    // validate job
    try {
        new ValidateJob(job).execute();
    } catch (CommandException e) {
        throw new IllegalMantisJobException(e);
    }
    // execute job
    List<StageConfig> stages = job.getStages();
    final SourceHolder source = job.getSource();
    final SinkHolder sink = job.getSink();
    final PortSelector portSelector = new PortSelectorInRange(8000, 9000);
    // register netty metrics
    RxNetty.useMetricListenersFactory(new MantisNettyEventsListenerFactory());
    // start our metrics server
    MetricsServer metricsServer = new MetricsServer(portSelector.acquirePort(), 1, Collections.EMPTY_MAP);
    metricsServer.start();
    Lifecycle lifecycle = job.getLifecycle();
    lifecycle.startup();
    // create job context
    Map parameterDefinitions = job.getParameterDefinitions();
    final String user = Optional.ofNullable(System.getenv("USER")).orElse("userUnknown");
    String jobId = String.format("localJob-%s-%d", user, (int) (Math.random() * 10000));
    logger.info("jobID {}", jobId);
    final ServiceLocator serviceLocator = lifecycle.getServiceLocator();
    int numInstances = schedulingInfo.forStage(1).getNumberOfInstances();
    BehaviorSubject<Integer> workersInStageOneObservable = BehaviorSubject.create(numInstances);
    BehaviorSubject<WorkerMap> workerMapObservable = BehaviorSubject.create();
    if (stages.size() == 1) {
        // single stage job
        final StageConfig stage = stages.get(0);
        // use latch to wait for all instances to complete
        final CountDownLatch waitUntilAllCompleted = new CountDownLatch(numInstances);
        Action0 countDownLatchOnComplete = new Action0() {

            @Override
            public void call() {
                waitUntilAllCompleted.countDown();
            }
        };
        Action0 nullOnCompleted = new Action0() {

            @Override
            public void call() {
            }
        };
        Action1<Throwable> nullOnError = new Action1<Throwable>() {

            @Override
            public void call(Throwable t) {
            }
        };
        Map<Integer, List<WorkerInfo>> workerInfoMap = new HashMap<>();
        List<WorkerInfo> workerInfoList = new ArrayList<>();
        // run for num of instances
        for (int i = 0; i < numInstances; i++) {
            WorkerPorts workerPorts = new WorkerPorts(portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort());
            WorkerInfo workerInfo = new WorkerInfo(jobId, jobId, 1, i, i + 1, MantisJobDurationType.Perpetual, "localhost", workerPorts);
            workerInfoList.add(workerInfo);
            Context context = new Context(ParameterUtils.createContextParameters(parameterDefinitions, parameters), lifecycle.getServiceLocator(), // new WorkerInfo(jobId, jobId, 1, i, i, MantisJobDurationType.Perpetual, "localhost", new ArrayList<>(),-1,-1),
            workerInfo, MetricsRegistry.getInstance(), () -> {
                System.exit(0);
            }, workerMapObservable);
            // workers for stage 1
            workerInfoMap.put(1, workerInfoList);
            workerMapObservable.onNext(new WorkerMap(workerInfoMap));
            StageExecutors.executeSingleStageJob(source, stage, sink, () -> workerInfo.getWorkerPorts().getSinkPort(), new RxMetrics(), context, countDownLatchOnComplete, i, workersInStageOneObservable, null, null, nullOnCompleted, nullOnError);
        }
        // wait for all instances to complete
        try {
            waitUntilAllCompleted.await();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    } else {
        // multi-stage job
        int workerNumber = 0;
        // start source stages
        StageConfig currentStage = stages.get(0);
        StageConfig previousStage = null;
        StageSchedulingInfo currentStageScalingInfo = schedulingInfo.forStage(1);
        StageSchedulingInfo nextStageScalingInfo = schedulingInfo.forStage(2);
        // num ports
        int[] previousPorts = new int[currentStageScalingInfo.getNumberOfInstances()];
        Map<Integer, List<WorkerInfo>> workerInfoMap = new HashMap<>();
        List<WorkerInfo> workerInfoList = new ArrayList<>();
        for (int i = 0; i < currentStageScalingInfo.getNumberOfInstances(); i++) {
            WorkerPorts workerPorts = new WorkerPorts(portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort());
            WorkerInfo workerInfo = new WorkerInfo(jobId, jobId, 1, i, i + 1, MantisJobDurationType.Perpetual, "localhost", workerPorts);
            workerInfoList.add(workerInfo);
            // int sourcePort = portSelector.acquirePort();
            int sourcePort = workerInfo.getWorkerPorts().getSinkPort();
            previousPorts[i] = sourcePort;
            Context context = new Context(ParameterUtils.createContextParameters(parameterDefinitions, parameters), serviceLocator, workerInfo, MetricsRegistry.getInstance(), nullAction, workerMapObservable);
            startSource(i, sourcePort, nextStageScalingInfo.getNumberOfInstances(), job.getSource(), currentStage, context, workersInStageOneObservable);
        }
        // workers for stage 1
        workerInfoMap.put(1, workerInfoList);
        workerMapObservable.onNext(new WorkerMap(workerInfoMap));
        // start intermediate stages, all but last stage
        for (int i = 1; i < stages.size() - 1; i++) {
            previousStage = currentStage;
            StageSchedulingInfo previousStageScalingInfo = schedulingInfo.forStage(i);
            // stages indexed starting at 1
            currentStageScalingInfo = schedulingInfo.forStage(i + 1);
            currentStage = stages.get(i);
            // stages indexed starting at 1
            nextStageScalingInfo = schedulingInfo.forStage(i + 2);
            int[] currentPorts = new int[currentStageScalingInfo.getNumberOfInstances()];
            workerInfoList = new ArrayList<>();
            for (int j = 0; j < currentStageScalingInfo.getNumberOfInstances(); j++) {
                WorkerPorts workerPorts = new WorkerPorts(portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort());
                WorkerInfo workerInfo = new WorkerInfo(jobId, jobId, i + 1, j, workerNumber++, MantisJobDurationType.Perpetual, "localhost", workerPorts);
                workerInfoList.add(workerInfo);
                // int port = portSelector.acquirePort();
                int port = workerInfo.getWorkerPorts().getSinkPort();
                currentPorts[j] = port;
                Context context = new Context(ParameterUtils.createContextParameters(parameterDefinitions, parameters), serviceLocator, workerInfo, MetricsRegistry.getInstance(), nullAction, workerMapObservable);
                startIntermediate(previousPorts, port, currentStage, context, j, nextStageScalingInfo.getNumberOfInstances(), i, previousStageScalingInfo.getNumberOfInstances());
            }
            // workers for current stage
            workerInfoMap.put(i + 1, workerInfoList);
            workerMapObservable.onNext(new WorkerMap(workerInfoMap));
            previousPorts = currentPorts;
        }
        // start sink stage
        StageSchedulingInfo previousStageScalingInfo = schedulingInfo.forStage(stages.size() - 1);
        previousStage = stages.get(stages.size() - 2);
        currentStage = stages.get(stages.size() - 1);
        currentStageScalingInfo = schedulingInfo.forStage(stages.size());
        numInstances = currentStageScalingInfo.getNumberOfInstances();
        // use latch to wait for all instances to complete
        final CountDownLatch waitUntilAllCompleted = new CountDownLatch(numInstances);
        Action0 countDownLatchOnTerminated = new Action0() {

            @Override
            public void call() {
                waitUntilAllCompleted.countDown();
            }
        };
        Action0 nullOnCompleted = new Action0() {

            @Override
            public void call() {
            }
        };
        Action1<Throwable> nullOnError = new Action1<Throwable>() {

            @Override
            public void call(Throwable t) {
            }
        };
        workerInfoList = new ArrayList<>();
        for (int i = 0; i < numInstances; i++) {
            WorkerPorts workerPorts = new WorkerPorts(portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort(), portSelector.acquirePort());
            WorkerInfo workerInfo = new WorkerInfo(jobId, jobId, stages.size(), i, workerNumber++, MantisJobDurationType.Perpetual, "localhost", workerPorts);
            workerInfoList.add(workerInfo);
            Context context = new Context(ParameterUtils.createContextParameters(parameterDefinitions, parameters), serviceLocator, workerInfo, MetricsRegistry.getInstance(), nullAction, workerMapObservable);
            startSink(previousStage, previousPorts, currentStage, () -> workerInfo.getWorkerPorts().getSinkPort(), sink, context, countDownLatchOnTerminated, nullOnCompleted, nullOnError, stages.size(), i, previousStageScalingInfo.getNumberOfInstances());
        }
        workerInfoMap.put(stages.size(), workerInfoList);
        workerMapObservable.onNext(new WorkerMap(workerInfoMap));
        // wait for all instances to complete
        try {
            waitUntilAllCompleted.await();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
    lifecycle.shutdown();
    metricsServer.shutdown();
}

Example 2 with WorkerMap

use of io.mantisrx.runtime.WorkerMap in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStageTest method convertJobSchedulingInfoToWorkerMapInvalidInputTest.

@Test
public void convertJobSchedulingInfoToWorkerMapInvalidInputTest() {
    String jobName = "convertJobSchedulingInfoToWorkerMapInvalidInputTest";
    String jobId = jobName + "-1";
    MantisJobDurationType durationType = MantisJobDurationType.Perpetual;
    WorkerAssignments workerAssignmentsStage1 = createWorkerAssignments(1, 2);
    WorkerAssignments workerAssignmentsStage2 = createWorkerAssignments(2, 4);
    Map<Integer, WorkerAssignments> workerAssignmentsMap = new HashMap<>();
    workerAssignmentsMap.put(1, workerAssignmentsStage1);
    workerAssignmentsMap.put(2, workerAssignmentsStage2);
    JobSchedulingInfo jobSchedulingInfo = new JobSchedulingInfo(jobId, workerAssignmentsMap);
    WorkerMap workerMap = WorkerExecutionOperationsNetworkStage.convertJobSchedulingInfoToWorkerMap(null, jobId, durationType, jobSchedulingInfo);
    assertTrue(workerMap.isEmpty());
    workerMap = WorkerExecutionOperationsNetworkStage.convertJobSchedulingInfoToWorkerMap(jobName, null, durationType, jobSchedulingInfo);
    assertTrue(workerMap.isEmpty());
    workerMap = WorkerExecutionOperationsNetworkStage.convertJobSchedulingInfoToWorkerMap(jobName, jobId, durationType, null);
    assertTrue(workerMap.isEmpty());
    jobSchedulingInfo = new JobSchedulingInfo(jobId, null);
    workerMap = WorkerExecutionOperationsNetworkStage.convertJobSchedulingInfoToWorkerMap(jobName, jobId, durationType, jobSchedulingInfo);
    assertTrue(workerMap.isEmpty());
    workerAssignmentsMap = new HashMap<>();
    workerAssignmentsMap.put(1, null);
    workerAssignmentsMap.put(2, workerAssignmentsStage2);
    jobSchedulingInfo = new JobSchedulingInfo(jobId, workerAssignmentsMap);
    workerMap = WorkerExecutionOperationsNetworkStage.convertJobSchedulingInfoToWorkerMap(jobName, jobId, durationType, jobSchedulingInfo);
    assertTrue(workerMap.isEmpty());
}

Example 3 with WorkerMap

use of io.mantisrx.runtime.WorkerMap in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStageTest method convertJobSchedulingInfoToWorkerMapTest.

@Test
public void convertJobSchedulingInfoToWorkerMapTest() {
    String jobName = "convertJobSchedulingInfoToWorkerMapTest";
    String jobId = jobName + "-1";
    MantisJobDurationType durationType = MantisJobDurationType.Perpetual;
    WorkerAssignments workerAssignmentsStage1 = createWorkerAssignments(1, 2);
    WorkerAssignments workerAssignmentsStage2 = createWorkerAssignments(2, 4);
    Map<Integer, WorkerAssignments> workerAssignmentsMap = new HashMap<>();
    workerAssignmentsMap.put(1, workerAssignmentsStage1);
    workerAssignmentsMap.put(2, workerAssignmentsStage2);
    JobSchedulingInfo jobSchedulingInfo = new JobSchedulingInfo(jobId, workerAssignmentsMap);
    WorkerMap workerMap = WorkerExecutionOperationsNetworkStage.convertJobSchedulingInfoToWorkerMap(jobName, jobId, durationType, jobSchedulingInfo);
    List<WorkerInfo> workersForStage1 = workerMap.getWorkersForStage(1);
    assertTrue(workersForStage1 != null);
    assertEquals(2, workersForStage1.size());
    for (int i = 0; i < workersForStage1.size(); i++) {
        WorkerInfo workerInfo = workersForStage1.get(i);
        assertEquals(i, workerInfo.getWorkerIndex());
        assertEquals(i + 1, workerInfo.getWorkerNumber());
        assertEquals(durationType, workerInfo.getDurationType());
        assertEquals(i, workerInfo.getWorkerPorts().getMetricsPort());
        assertEquals(i + 1, workerInfo.getWorkerPorts().getCustomPort());
    }
    List<WorkerInfo> workersForStage2 = workerMap.getWorkersForStage(2);
    assertTrue(workersForStage2 != null);
    assertEquals(4, workersForStage2.size());
    for (int i = 0; i < workersForStage2.size(); i++) {
        WorkerInfo workerInfo = workersForStage2.get(i);
        assertEquals(i, workerInfo.getWorkerIndex());
        assertEquals(i + 1, workerInfo.getWorkerNumber());
        assertEquals(durationType, workerInfo.getDurationType());
        assertEquals(i, workerInfo.getWorkerPorts().getMetricsPort());
        assertEquals(i + 1, workerInfo.getWorkerPorts().getCustomPort());
    }
}

Example 4 with WorkerMap

use of io.mantisrx.runtime.WorkerMap in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method convertJobSchedulingInfoToWorkerMap.

/**
 * Converts a JobSchedulingInfo object to a simple WorkerMap to be used from within the context.
 * Static for easier testing.
 *
 * @param jobName
 * @param jobId
 * @param durationType
 * @param js
 *
 * @return
 */
static WorkerMap convertJobSchedulingInfoToWorkerMap(String jobName, String jobId, MantisJobDurationType durationType, JobSchedulingInfo js) {
    Map<Integer, List<WorkerInfo>> stageToWorkerInfoMap = new HashMap<>();
    WorkerMap workerMap = new WorkerMap(stageToWorkerInfoMap);
    if (jobName == null || jobName.isEmpty() || jobId == null || jobId.isEmpty()) {
        logger.warn("Job name/jobId cannot be null in convertJobSchedulingInfoToWorkerMap");
        return workerMap;
    }
    if (js == null || js.getWorkerAssignments() == null) {
        logger.warn("JobSchedulingInfo or workerAssignments cannot be null in convertJobSchedulingInfoToWorkerMap");
        return workerMap;
    }
    try {
        Map<Integer, WorkerAssignments> workerAssignments = js.getWorkerAssignments();
        Iterator<Map.Entry<Integer, WorkerAssignments>> entryIterator = workerAssignments.entrySet().iterator();
        while (entryIterator.hasNext()) {
            Map.Entry<Integer, WorkerAssignments> next = entryIterator.next();
            int stageNo = next.getKey();
            WorkerAssignments workerAssignmentsForStage = next.getValue();
            Map<Integer, WorkerHost> hosts = workerAssignmentsForStage.getHosts();
            if (hosts != null) {
                List<WorkerInfo> workerInfoList = hosts.values().stream().map((workerHost) -> {
                    return generateWorkerInfo(jobName, jobId, stageNo, workerHost.getWorkerIndex(), workerHost.getWorkerNumber(), durationType, workerHost.getHost(), workerHost);
                }).collect(Collectors.toList());
                stageToWorkerInfoMap.put(stageNo, workerInfoList);
            }
        }
        workerMap = new WorkerMap(stageToWorkerInfoMap);
    } catch (Exception e) {
        logger.warn("Exception converting JobSchedulingInfo " + js + " to worker Map " + e.getMessage());
        return workerMap;
    }
    return workerMap;
}