Examples with JobDefinition io.mantisrx.server.master.domain.JobDefinition used on opensource projects

Example 1 with JobDefinition

use of io.mantisrx.server.master.domain.JobDefinition in project mantis by Netflix.

the class JobDefinitionResolverTest method artifactPresentButSchedAbsentFailsTest.

@Test
public void artifactPresentButSchedAbsentFailsTest() {
    String clusterName = "artifactPresentButSchedAbsentFailsTest";
    List<Label> labels = new ArrayList<>();
    Label label = new Label("l1", "lv1");
    labels.add(label);
    List<Parameter> parameters = new ArrayList<>();
    Parameter parameter = new Parameter("paramName", "paramValue");
    parameters.add(parameter);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, labels, parameters);
    IJobClusterMetadata jobClusterMetadata = new JobClusterMetadataImpl(fakeJobCluster, 1, false);
    String version = "0.0.2";
    String artifactName = "myArt2";
    // Only new artifact and version is specified
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withArtifactName(artifactName).withName(clusterName).withVersion(version).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        fail();
    } catch (Exception e) {
        e.printStackTrace();
    }
    // Only new artifact is specified
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withArtifactName(artifactName).withName(clusterName).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        fail();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 2 with JobDefinition

use of io.mantisrx.server.master.domain.JobDefinition in project mantis by Netflix.

the class JobDefinitionResolverTest method versionNotFoundTest.

@Test
public void versionNotFoundTest() {
    String clusterName = "versionNotFoundTest";
    List<Label> labels = new ArrayList<>();
    Label label = new Label("l1", "lv1");
    labels.add(label);
    List<Parameter> parameters = new ArrayList<>();
    Parameter parameter = new Parameter("paramName", "paramValue");
    parameters.add(parameter);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, labels, parameters);
    IJobClusterMetadata jobClusterMetadata = new JobClusterMetadataImpl(fakeJobCluster, 1, false);
    String version = "0.0.2";
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withName(clusterName).withVersion(version).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        fail();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 3 with JobDefinition

use of io.mantisrx.server.master.domain.JobDefinition in project mantis by Netflix.

the class TestHelpers method createFakeScheduleRequest.

public static ScheduleRequest createFakeScheduleRequest(final WorkerId workerId, final int stageNum, final int numStages, final MachineDefinition machineDefinition) {
    try {
        JobDefinition jobDefinition = new JobDefinition.Builder().withArtifactName("jar").withSchedulingInfo(new SchedulingInfo(Collections.singletonMap(0, StageSchedulingInfo.builder().numberOfInstances(1).machineDefinition(machineDefinition).hardConstraints(Collections.emptyList()).softConstraints(Collections.emptyList()).build()))).withJobSla(new JobSla(0, 0, null, MantisJobDurationType.Perpetual, null)).build();
        IMantisJobMetadata mantisJobMetadata = new MantisJobMetadataImpl.Builder().withJobId(JobId.fromId(workerId.getJobId()).get()).withJobDefinition(jobDefinition).build();
        return new ScheduleRequest(workerId, stageNum, numStages, new JobMetadata(mantisJobMetadata.getJobId().getId(), mantisJobMetadata.getJobJarUrl(), mantisJobMetadata.getTotalStages(), mantisJobMetadata.getUser(), mantisJobMetadata.getSchedulingInfo(), mantisJobMetadata.getParameters(), mantisJobMetadata.getSubscriptionTimeoutSecs(), mantisJobMetadata.getMinRuntimeSecs()), mantisJobMetadata.getSla().get().getDurationType(), machineDefinition, Collections.emptyList(), Collections.emptyList(), 0, Optional.empty());
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}

Example 4 with JobDefinition

use of io.mantisrx.server.master.domain.JobDefinition in project mantis by Netflix.

the class JobClusterActor method onGetJobDefinitionUpdatedFromJobActorResponse.

public void onGetJobDefinitionUpdatedFromJobActorResponse(GetJobDefinitionUpdatedFromJobActorResponse request) {
    logger.info("Resuming job submission from job actor");
    // this request is returned by job actor but the response needs to be replied to the original job request sender (from API routes).
    ActorRef originalSender = request.getOriginalSender();
    if (request.responseCode == SERVER_ERROR || request.getJobDefinition() == null) {
        logger.error("Failed to retrieve job definition from job actor");
        numJobSubmissionFailures.increment();
        originalSender.tell(new SubmitJobResponse(request.requestId, SERVER_ERROR, request.message, empty()), getSelf());
        return;
    }
    try {
        JobDefinition resolvedJobDefn = request.getJobDefinition();
        // for quick submit the artifact version/name needs to be reset using the fork method below.
        if (request.isQuickSubmitMode()) {
            Optional<JobDefinition> jobDefinitionCloneO = cloneToNewJobDefinitionWithoutArtifactNameAndVersion(request.getJobDefinition());
            if (jobDefinitionCloneO.isPresent()) {
                resolvedJobDefn = jobDefinitionCloneO.get();
            }
        }
        resolvedJobDefn = this.jobDefinitionResolver.getResolvedJobDefinition(request.getUser(), resolvedJobDefn, this.jobClusterMetadata);
        eventPublisher.publishStatusEvent(new LifecycleEventsProto.JobClusterStatusEvent(LifecycleEventsProto.StatusEvent.StatusEventType.INFO, "Job submit request received", jobClusterMetadata.getJobClusterDefinition().getName()));
        resolvedJobDefn = LabelManager.insertSystemLabels(resolvedJobDefn, request.isAutoResubmit());
        submitJob(resolvedJobDefn, originalSender, request.getUser());
        numJobSubmissions.increment();
    } catch (PersistException pe) {
        logger.error("Exception submitting job {} from {}", this.name, request.getUser(), pe);
        numJobSubmissionFailures.increment();
        originalSender.tell(new SubmitJobResponse(request.requestId, SERVER_ERROR, pe.getMessage(), empty()), getSelf());
    } catch (Exception e) {
        logger.error("Exception submitting job {} from {}", this.name, request.getUser(), e);
        numJobSubmissionFailures.increment();
        originalSender.tell(new SubmitJobResponse(request.requestId, CLIENT_ERROR, e.getMessage(), empty()), getSelf());
    }
}

Example 5 with JobDefinition

use of io.mantisrx.server.master.domain.JobDefinition in project mantis by Netflix.

the class JobClusterActor method getResolvedJobDefinition.

/**
 * Two cases
 * 1. JobDefinition provided by user: In this case check if labels / parameters or schedulingInfo was not provided
 * if that is the case inherit from the Cluster
 * 2. If JobDefinition is not provided, find the last submitted job and use its config (quick submit)
 * @param user submitter
 * @param givenJobDefnOp job defn provided by user in job submit
 * @return job definition to be used by the actual submit
 * @throws Exception If jobDefinition could not be resolved
 */
private JobDefinition getResolvedJobDefinition(final String user, final Optional<JobDefinition> givenJobDefnOp) throws Exception {
    JobDefinition resolvedJobDefn;
    if (givenJobDefnOp.isPresent()) {
        if (givenJobDefnOp.get().getSchedulingInfo() != null && givenJobDefnOp.get().requireInheritInstanceCheck()) {
            logger.warn("Job requires inheriting instance count but has no active non-terminal job.");
        }
        resolvedJobDefn = givenJobDefnOp.get();
    } else {
        // no job definition specified , this is quick submit which is supposed to inherit from last job submitted
        // for request inheriting from non-terminal jobs, it has been sent to job actor instead.
        Optional<JobDefinition> jobDefnOp = cloneJobDefinitionForQuickSubmitFromArchivedJobs(jobManager.getCompletedJobsList(), empty(), jobStore);
        if (jobDefnOp.isPresent()) {
            logger.info("Inherited scheduling Info and parameters from previous job");
            resolvedJobDefn = jobDefnOp.get();
        } else if (this.jobClusterMetadata != null && this.jobClusterMetadata.getJobClusterDefinition() != null && this.jobClusterMetadata.getJobClusterDefinition().getJobClusterConfig() != null) {
            logger.info("No previous job definition found. Fall back to cluster definition: {}", this.name);
            IJobClusterDefinition clusterDefinition = this.jobClusterMetadata.getJobClusterDefinition();
            JobClusterConfig clusterConfig = this.jobClusterMetadata.getJobClusterDefinition().getJobClusterConfig();
            resolvedJobDefn = new JobDefinition.Builder().withJobSla(new JobSla.Builder().build()).withArtifactName(clusterConfig.getArtifactName()).withVersion(clusterConfig.getVersion()).withLabels(clusterDefinition.getLabels()).withName(this.name).withParameters(clusterDefinition.getParameters()).withSchedulingInfo(clusterConfig.getSchedulingInfo()).withUser(user).build();
            logger.info("Built job definition from cluster definition: {}", resolvedJobDefn);
        } else {
            throw new Exception("Job Definition could not retrieved from a previous submission (There may " + "not be a previous submission)");
        }
    }
    logger.info("Resolved JobDefn {}", resolvedJobDefn);
    return this.jobDefinitionResolver.getResolvedJobDefinition(user, resolvedJobDefn, this.jobClusterMetadata);
}