Examples with StageSchedulingInfo io.mantisrx.runtime.descriptor.StageSchedulingInfo used on opensource projects

Example 1 with StageSchedulingInfo

use of io.mantisrx.runtime.descriptor.StageSchedulingInfo in project mantis by Netflix.

the class MantisJobDefinition method validateSchedulingInfo.

private void validateSchedulingInfo(boolean schedulingInfoOptional) throws InvalidJobException {
    if (schedulingInfoOptional && schedulingInfo == null)
        return;
    if (schedulingInfo == null)
        throw new InvalidJobException("No scheduling info provided");
    if (schedulingInfo.getStages() == null)
        throw new InvalidJobException("No stages defined in scheduling info");
    int numStages = schedulingInfo.getStages().size();
    int startingIdx = 1;
    if (schedulingInfo.forStage(0) != null) {
        // jobMaster stage 0 definition exists, adjust index range
        startingIdx = 0;
        numStages--;
    }
    for (int i = startingIdx; i <= numStages; i++) {
        StageSchedulingInfo stage = schedulingInfo.getStages().get(i);
        if (stage == null)
            throw new InvalidJobException("No definition for stage " + i + " in scheduling info for " + numStages + " stage job");
        if (stage.getNumberOfInstances() < 1)
            throw new InvalidJobException("Number of instance for stage " + i + " must be >0, not " + stage.getNumberOfInstances());
        MachineDefinition machineDefinition = stage.getMachineDefinition();
        if (machineDefinition.getCpuCores() <= 0)
            throw new InvalidJobException("cpuCores must be >0.0, not " + machineDefinition.getCpuCores());
        if (machineDefinition.getMemoryMB() <= 0)
            throw new InvalidJobException("memory must be <0.0, not " + machineDefinition.getMemoryMB());
        if (machineDefinition.getDiskMB() < 0)
            throw new InvalidJobException("disk must be >=0, not " + machineDefinition.getDiskMB());
        if (machineDefinition.getNumPorts() < 0)
            throw new InvalidJobException("numPorts must be >=0, not " + machineDefinition.getNumPorts());
    }
}

Example 2 with StageSchedulingInfo

use of io.mantisrx.runtime.descriptor.StageSchedulingInfo in project mantis by Netflix.

the class JobActor method isAutoscaled.

private boolean isAutoscaled(SchedulingInfo schedulingInfo) {
    LOGGER.trace("In isAutoscaled {}", schedulingInfo);
    for (Map.Entry<Integer, StageSchedulingInfo> entry : schedulingInfo.getStages().entrySet()) {
        final StageScalingPolicy scalingPolicy = entry.getValue().getScalingPolicy();
        if (scalingPolicy != null && scalingPolicy.isEnabled()) {
            LOGGER.info("Job {} is autoscaleable", jobId);
            return true;
        }
    }
    LOGGER.info("Job {} is NOT scaleable", jobId);
    return false;
}

Example 3 with StageSchedulingInfo

use of io.mantisrx.runtime.descriptor.StageSchedulingInfo in project mantis by Netflix.

the class JobsRoute method validateSubmitJobRequest.

/**
 * @return true to indicate valid, false otherwise. The String holds the error message when the request is invalid
 */
private Pair<Boolean, String> validateSubmitJobRequest(MantisJobDefinition mjd, Optional<String> clusterNameInResource) {
    if (null == mjd) {
        logger.error("rejecting job submit request, job definition is malformed {}", mjd);
        return Pair.apply(false, "Malformed job definition.");
    }
    // must include job cluster name
    if (mjd.getName() == null || mjd.getName().length() == 0) {
        logger.info("rejecting job submit request, must include name {}", mjd);
        return Pair.apply(false, "Job definition must include name");
    }
    // validate specified job cluster name matches with what specified in REST resource endpoint
    if (clusterNameInResource.isPresent()) {
        if (!clusterNameInResource.get().equals(mjd.getName())) {
            String msg = String.format("Cluster name specified in request payload [%s] " + "does not match with what specified in resource endpoint [%s]", mjd.getName(), clusterNameInResource.get());
            logger.info("rejecting job submit request, {} {}", msg, mjd);
            return Pair.apply(false, msg);
        }
    }
    // validate scheduling info
    SchedulingInfo schedulingInfo = mjd.getSchedulingInfo();
    if (schedulingInfo != null) {
        Map<Integer, StageSchedulingInfo> stages = schedulingInfo.getStages();
        if (stages != null) {
            for (StageSchedulingInfo stageSchedInfo : stages.values()) {
                double cpuCores = stageSchedInfo.getMachineDefinition().getCpuCores();
                int maxCpuCores = ConfigurationProvider.getConfig().getWorkerMachineDefinitionMaxCpuCores();
                if (cpuCores > maxCpuCores) {
                    logger.info("rejecting job submit request, requested CPU {} > max for {} (user: {}) (stage: {})", cpuCores, mjd.getName(), mjd.getUser(), stages);
                    return Pair.apply(false, "requested CPU cannot be more than max CPU per worker " + maxCpuCores);
                }
                double memoryMB = stageSchedInfo.getMachineDefinition().getMemoryMB();
                int maxMemoryMB = ConfigurationProvider.getConfig().getWorkerMachineDefinitionMaxMemoryMB();
                if (memoryMB > maxMemoryMB) {
                    logger.info("rejecting job submit request, requested memory {} > max for {} (user: {}) (stage: {})", memoryMB, mjd.getName(), mjd.getUser(), stages);
                    return Pair.apply(false, "requested memory cannot be more than max memoryMB per worker " + maxMemoryMB);
                }
                double networkMbps = stageSchedInfo.getMachineDefinition().getNetworkMbps();
                int maxNetworkMbps = ConfigurationProvider.getConfig().getWorkerMachineDefinitionMaxNetworkMbps();
                if (networkMbps > maxNetworkMbps) {
                    logger.info("rejecting job submit request, requested network {} > max for {} (user: {}) (stage: {})", networkMbps, mjd.getName(), mjd.getUser(), stages);
                    return Pair.apply(false, "requested network cannot be more than max networkMbps per worker " + maxNetworkMbps);
                }
                int numberOfInstances = stageSchedInfo.getNumberOfInstances();
                int maxWorkersPerStage = ConfigurationProvider.getConfig().getMaxWorkersPerStage();
                if (numberOfInstances > maxWorkersPerStage) {
                    logger.info("rejecting job submit request, requested num instances {} > max for {} (user: {}) (stage: {})", numberOfInstances, mjd.getName(), mjd.getUser(), stages);
                    return Pair.apply(false, "requested number of instances per stage cannot be more than " + maxWorkersPerStage);
                }
                StageScalingPolicy scalingPolicy = stageSchedInfo.getScalingPolicy();
                if (scalingPolicy != null) {
                    if (scalingPolicy.getMax() > maxWorkersPerStage) {
                        logger.info("rejecting job submit request, requested num instances in scaling policy {} > max for {} (user: {}) (stage: {})", numberOfInstances, mjd.getName(), mjd.getUser(), stages);
                        return Pair.apply(false, "requested number of instances per stage in scaling policy cannot be more than " + maxWorkersPerStage);
                    }
                }
            }
        }
    }
    return Pair.apply(true, "");
}

Example 4 with StageSchedulingInfo

use of io.mantisrx.runtime.descriptor.StageSchedulingInfo in project mantis by Netflix.

the class JobDefinition method validateSchedulingInfo.

private void validateSchedulingInfo(boolean schedulingInfoOptional) throws InvalidJobException {
    if (schedulingInfoOptional && schedulingInfo == null)
        return;
    if (schedulingInfo == null)
        throw new InvalidJobException("No scheduling info provided");
    if (schedulingInfo.getStages() == null)
        throw new InvalidJobException("No stages defined in scheduling info");
    int withNumberOfStages = schedulingInfo.getStages().size();
    int startingIdx = 1;
    if (schedulingInfo.forStage(0) != null) {
        // jobMaster stage 0 definition exists, adjust index range
        startingIdx = 0;
        withNumberOfStages--;
    }
    for (int i = startingIdx; i <= withNumberOfStages; i++) {
        StageSchedulingInfo stage = schedulingInfo.getStages().get(i);
        if (stage == null)
            throw new InvalidJobException("No definition for stage " + i + " in scheduling info for " + withNumberOfStages + " stage job");
        if (stage.getNumberOfInstances() < 1)
            throw new InvalidJobException("Number of instance for stage " + i + " must be >0, not " + stage.getNumberOfInstances());
        MachineDefinition machineDefinition = stage.getMachineDefinition();
        if (machineDefinition.getCpuCores() <= 0)
            throw new InvalidJobException("cpuCores must be >0.0, not " + machineDefinition.getCpuCores());
        if (machineDefinition.getMemoryMB() <= 0)
            throw new InvalidJobException("memory must be <0.0, not " + machineDefinition.getMemoryMB());
        if (machineDefinition.getDiskMB() < 0)
            throw new InvalidJobException("disk must be >=0, not " + machineDefinition.getDiskMB());
        if (machineDefinition.getNumPorts() < 0)
            throw new InvalidJobException("numPorts must be >=0, not " + machineDefinition.getNumPorts());
    }
}

Example 5 with StageSchedulingInfo

use of io.mantisrx.runtime.descriptor.StageSchedulingInfo in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method executeStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void executeStage(final ExecutionDetails setup) {
    ExecuteStageRequest executionRequest = setup.getExecuteStageRequest().getRequest();
    // Initialize the schedulingInfo observable for current job and mark it shareable to be reused by anyone interested in this data.
    // Observable<JobSchedulingInfo> selfSchedulingInfo = mantisMasterApi.schedulingChanges(executionRequest.getJobId()).switchMap((e) -> Observable.just(e).repeatWhen(x -> x.delay(5 , TimeUnit.SECONDS))).subscribeOn(Schedulers.io()).share();
    Observable<JobSchedulingInfo> selfSchedulingInfo = mantisMasterApi.schedulingChanges(executionRequest.getJobId()).subscribeOn(Schedulers.io()).share();
    WorkerInfo workerInfo = generateWorkerInfo(executionRequest.getJobName(), executionRequest.getJobId(), executionRequest.getStage(), executionRequest.getWorkerIndex(), executionRequest.getWorkerNumber(), executionRequest.getDurationType(), "host", executionRequest.getWorkerPorts());
    final Observable<Integer> sourceStageTotalWorkersObs = createSourceStageTotalWorkersObservable(selfSchedulingInfo);
    RunningWorker.Builder rwBuilder = new RunningWorker.Builder().job(setup.getMantisJob()).schedulingInfo(executionRequest.getSchedulingInfo()).stageTotalWorkersObservable(sourceStageTotalWorkersObs).jobName(executionRequest.getJobName()).stageNum(executionRequest.getStage()).workerIndex(executionRequest.getWorkerIndex()).workerNum(executionRequest.getWorkerNumber()).totalStages(executionRequest.getTotalNumStages()).metricsPort(executionRequest.getMetricsPort()).ports(executionRequest.getPorts().iterator()).jobStatusObserver(setup.getStatus()).requestSubject(setup.getExecuteStageRequest().getRequestSubject()).workerInfo(workerInfo).vmTaskStatusObservable(vmTaskStatusObserver).hasJobMaster(executionRequest.getHasJobMaster()).jobId(executionRequest.getJobId());
    if (executionRequest.getStage() == 0) {
        rwBuilder = rwBuilder.stage(new JobMasterStageConfig("jobmasterconfig"));
    } else {
        rwBuilder = rwBuilder.stage((StageConfig) setup.getMantisJob().getStages().get(executionRequest.getStage() - 1));
    }
    final RunningWorker rw = rwBuilder.build();
    AtomicReference<SubscriptionStateHandler> subscriptionStateHandlerRef = new AtomicReference<>();
    if (rw.getStageNum() == rw.getTotalStagesNet()) {
        // set up subscription state handler only for sink (last) stage
        subscriptionStateHandlerRef.set(setupSubscriptionStateHandler(setup.getExecuteStageRequest().getRequest().getJobId(), mantisMasterApi, setup.getExecuteStageRequest().getRequest().getSubscriptionTimeoutSecs(), setup.getExecuteStageRequest().getRequest().getMinRuntimeSecs()));
    }
    logger.info("Running worker info: " + rw);
    rw.signalStartedInitiated();
    try {
        logger.info(">>>>>>>>>>>>>>>>Calling lifecycle.startup()");
        Lifecycle lifecycle = rw.getJob().getLifecycle();
        lifecycle.startup();
        ServiceLocator serviceLocator = lifecycle.getServiceLocator();
        if (lookupSpectatorRegistry) {
            try {
                final Registry spectatorRegistry = serviceLocator.service(Registry.class);
                SpectatorRegistryFactory.setRegistry(spectatorRegistry);
            } catch (Throwable t) {
                logger.error("failed to init spectator registry using service locator, falling back to {}", SpectatorRegistryFactory.getRegistry().getClass().getCanonicalName());
            }
        }
        // create job context
        Parameters parameters = ParameterUtils.createContextParameters(rw.getJob().getParameterDefinitions(), setup.getParameters());
        final Context context = generateContext(parameters, serviceLocator, workerInfo, MetricsRegistry.getInstance(), () -> {
            rw.signalCompleted();
            // wait for completion signal to go to the master and us getting killed. Upon timeout, exit.
            try {
                Thread.sleep(60000);
            } catch (InterruptedException ie) {
                logger.warn("Unexpected exception sleeping: " + ie.getMessage());
            }
            System.exit(0);
        }, createWorkerMapObservable(selfSchedulingInfo, executionRequest.getJobName(), executionRequest.getJobId(), executionRequest.getDurationType()));
        // context.setPrevStageCompletedObservable(createPrevStageCompletedObservable(selfSchedulingInfo, rw.getJobId(), rw.getStageNum()));
        rw.setContext(context);
        // setup heartbeats
        heartbeatRef.set(new Heartbeat(rw.getJobId(), rw.getStageNum(), rw.getWorkerIndex(), rw.getWorkerNum()));
        final double networkMbps = executionRequest.getSchedulingInfo().forStage(rw.getStageNum()).getMachineDefinition().getNetworkMbps();
        startSendingHeartbeats(rw.getJobStatus(), new WorkerId(executionRequest.getJobId(), executionRequest.getWorkerIndex(), executionRequest.getWorkerNumber()).getId(), networkMbps);
        // execute stage
        if (rw.getStageNum() == 0) {
            logger.info("JobId: " + rw.getJobId() + ", executing Job Master");
            final AutoScaleMetricsConfig autoScaleMetricsConfig = new AutoScaleMetricsConfig();
            // Temporary workaround to enable auto-scaling by custom metric in Job Master. This will be revisited to get the entire autoscaling config
            // for a job as a System parameter in the JobMaster
            final String autoScaleMetricString = (String) parameters.get(JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM, "");
            if (!Strings.isNullOrEmpty(autoScaleMetricString)) {
                final List<String> tokens = Splitter.on("::").omitEmptyStrings().trimResults().splitToList(autoScaleMetricString);
                if (tokens.size() == 3) {
                    final String metricGroup = tokens.get(0);
                    final String metricName = tokens.get(1);
                    final String algo = tokens.get(2);
                    try {
                        final AutoScaleMetricsConfig.AggregationAlgo aggregationAlgo = AutoScaleMetricsConfig.AggregationAlgo.valueOf(algo);
                        logger.info("registered UserDefined auto scale metric {}:{} algo {}", metricGroup, metricName, aggregationAlgo);
                        autoScaleMetricsConfig.addUserDefinedMetric(metricGroup, metricName, aggregationAlgo);
                    } catch (IllegalArgumentException e) {
                        final String errorMsg = String.format("ERROR: Invalid algorithm value %s for param %s (algo should be one of %s)", autoScaleMetricsConfig, JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM, Arrays.stream(AutoScaleMetricsConfig.AggregationAlgo.values()).map(a -> a.name()).collect(Collectors.toList()));
                        logger.error(errorMsg);
                        throw new RuntimeException(errorMsg);
                    }
                } else {
                    final String errorMsg = String.format("ERROR: Invalid value %s for param %s", autoScaleMetricString, JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM);
                    logger.error(errorMsg);
                    throw new RuntimeException(errorMsg);
                }
            } else {
                logger.info("param {} is null or empty", JOB_MASTER_AUTOSCALE_METRIC_SYSTEM_PARAM);
            }
            final JobMasterService jobMasterService = new JobMasterService(rw.getJobId(), rw.getSchedulingInfo(), workerMetricsClient, autoScaleMetricsConfig, mantisMasterApi, rw.getContext(), rw.getOnCompleteCallback(), rw.getOnErrorCallback(), rw.getOnTerminateCallback());
            jobMasterService.start();
            signalStarted(rw, subscriptionStateHandlerRef);
            // block until worker terminates
            rw.waitUntilTerminate();
        } else if (rw.getStageNum() == 1 && rw.getTotalStagesNet() == 1) {
            logger.info("JobId: " + rw.getJobId() + ", single stage job, executing entire job");
            // single stage, execute entire job on this machine
            PortSelector portSelector = new PortSelector() {

                @Override
                public int acquirePort() {
                    return rw.getPorts().next();
                }
            };
            RxMetrics rxMetrics = new RxMetrics();
            StageExecutors.executeSingleStageJob(rw.getJob().getSource(), rw.getStage(), rw.getJob().getSink(), portSelector, rxMetrics, rw.getContext(), rw.getOnTerminateCallback(), rw.getWorkerIndex(), rw.getSourceStageTotalWorkersObservable(), onSinkSubscribe, onSinkUnsubscribe, rw.getOnCompleteCallback(), rw.getOnErrorCallback());
            signalStarted(rw, subscriptionStateHandlerRef);
            // block until worker terminates
            rw.waitUntilTerminate();
        } else {
            logger.info("JobId: " + rw.getJobId() + ", executing a multi-stage job, stage: " + rw.getStageNum());
            if (rw.getStageNum() == 1) {
                // execute source stage
                String remoteObservableName = rw.getJobId() + "_" + rw.getStageNum();
                StageSchedulingInfo currentStageSchedulingInfo = rw.getSchedulingInfo().forStage(1);
                WorkerPublisherRemoteObservable publisher = new WorkerPublisherRemoteObservable<>(rw.getPorts().next(), remoteObservableName, numWorkersAtStage(selfSchedulingInfo, rw.getJobId(), rw.getStageNum() + 1), rw.getJobName());
                StageExecutors.executeSource(rw.getWorkerIndex(), rw.getJob().getSource(), rw.getStage(), publisher, rw.getContext(), rw.getSourceStageTotalWorkersObservable());
                logger.info("JobId: " + rw.getJobId() + " stage: " + rw.getStageNum() + ", serving remote observable for source with name: " + remoteObservableName);
                RemoteRxServer server = publisher.getServer();
                RxMetrics rxMetrics = server.getMetrics();
                MetricsRegistry.getInstance().registerAndGet(rxMetrics.getCountersAndGauges());
                signalStarted(rw, subscriptionStateHandlerRef);
                logger.info("JobId: " + rw.getJobId() + " stage: " + rw.getStageNum() + ", blocking until source observable completes");
                server.blockUntilServerShutdown();
            } else {
                // execute intermediate stage or last stage plus sink
                executeNonSourceStage(selfSchedulingInfo, rw, subscriptionStateHandlerRef);
            }
        }
        logger.info("Calling lifecycle.shutdown()");
        lifecycle.shutdown();
    } catch (Throwable t) {
        rw.signalFailed(t);
        shutdownStage();
    }
}