Examples with InvalidJobException io.mantisrx.runtime.command.InvalidJobException used on opensource projects

Example 11 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobDefinitionResolverTest method SchedPresentTest.

@Test
public void SchedPresentTest() {
    String clusterName = "SchedPresentTest";
    List<Label> labels = new ArrayList<>();
    Label label = new Label("l1", "lv1");
    labels.add(label);
    List<Parameter> parameters = new ArrayList<>();
    Parameter parameter = new Parameter("paramName", "paramValue");
    parameters.add(parameter);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, labels, parameters);
    IJobClusterMetadata jobClusterMetadata = new JobClusterMetadataImpl(fakeJobCluster, 1, false);
    JobConstraints softConstraints = JobConstraints.ExclusiveHost;
    List<JobConstraints> constraintsList = new ArrayList<>();
    constraintsList.add(softConstraints);
    SchedulingInfo schedulingInfo = new SchedulingInfo.Builder().numberOfStages(1).singleWorkerStageWithConstraints(DEFAULT_MACHINE_DEFINITION, Lists.newArrayList(), constraintsList).build();
    try {
        // only sched info set.
        JobDefinition givenJobDefn = new JobDefinition.Builder().withName(clusterName).withSchedulingInfo(schedulingInfo).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        // artifact will get populated using the given version.
        assertEquals(DEFAULT_ARTIFACT_NAME, resolvedJobDefinition.getArtifactName());
        // scheduling info will be the one specified by us
        assertEquals(schedulingInfo, resolvedJobDefinition.getSchedulingInfo());
        // version should match the latest on the cluster
        assertEquals(DEFAULT_VERSION, resolvedJobDefinition.getVersion());
        // assert the parameters and labels are inherited since they were not specified
        assertEquals(1, resolvedJobDefinition.getLabels().size());
        assertEquals(label, resolvedJobDefinition.getLabels().get(0));
        assertEquals(1, resolvedJobDefinition.getParameters().size());
        assertEquals(parameter, resolvedJobDefinition.getParameters().get(0));
    } catch (InvalidJobException e) {
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
    // NOTHING is specified
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withName(clusterName).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        // assert the artifact is inherited
        assertEquals(DEFAULT_ARTIFACT_NAME, resolvedJobDefinition.getArtifactName());
        // assert the scheduling info is inherited
        assertEquals(SINGLE_WORKER_SCHED_INFO, resolvedJobDefinition.getSchedulingInfo());
        // assert a version is the dfeault one.
        assertEquals(DEFAULT_VERSION, resolvedJobDefinition.getVersion());
        // assert the parameters and labels are inherited since they were not specified
        assertEquals(1, resolvedJobDefinition.getLabels().size());
        assertEquals(label, resolvedJobDefinition.getLabels().get(0));
        assertEquals(1, resolvedJobDefinition.getParameters().size());
        assertEquals(parameter, resolvedJobDefinition.getParameters().get(0));
    } catch (InvalidJobException e) {
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
    // NOTHING is specified2
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withName(clusterName).withVersion("null").build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        // assert the artifact is inherited
        assertEquals(DEFAULT_ARTIFACT_NAME, resolvedJobDefinition.getArtifactName());
        // assert the scheduling info is inherited
        assertEquals(SINGLE_WORKER_SCHED_INFO, resolvedJobDefinition.getSchedulingInfo());
        // assert a version is the dfeault one.
        assertEquals(DEFAULT_VERSION, resolvedJobDefinition.getVersion());
        // assert the parameters and labels are inherited since they were not specified
        assertEquals(1, resolvedJobDefinition.getLabels().size());
        assertEquals(label, resolvedJobDefinition.getLabels().get(0));
        assertEquals(1, resolvedJobDefinition.getParameters().size());
        assertEquals(parameter, resolvedJobDefinition.getParameters().get(0));
    } catch (InvalidJobException e) {
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 12 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobDefinitionResolverTest method artifactSchedPresentTest.

@Test
public void artifactSchedPresentTest() {
    String clusterName = "artifactVersionSchedPresentTest";
    List<Label> labels = new ArrayList<>();
    Label label = new Label("l1", "lv1");
    labels.add(label);
    List<Parameter> parameters = new ArrayList<>();
    Parameter parameter = new Parameter("paramName", "paramValue");
    parameters.add(parameter);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, labels, parameters);
    IJobClusterMetadata jobClusterMetadata = new JobClusterMetadataImpl(fakeJobCluster, 1, false);
    String version = "0.0.2";
    String artifactName = "myArt2";
    SchedulingInfo schedulingInfo = TWO_WORKER_SCHED_INFO;
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withArtifactName(artifactName).withName(clusterName).withSchedulingInfo(schedulingInfo).withVersion(version).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        // assert the specified values are being used
        assertEquals(artifactName, resolvedJobDefinition.getArtifactName());
        assertEquals(schedulingInfo, resolvedJobDefinition.getSchedulingInfo());
        assertEquals(version, resolvedJobDefinition.getVersion());
        // assert the parameters and labels are inherited since they were not specified
        assertEquals(1, resolvedJobDefinition.getLabels().size());
        assertEquals(label, resolvedJobDefinition.getLabels().get(0));
        assertEquals(1, resolvedJobDefinition.getParameters().size());
        assertEquals(parameter, resolvedJobDefinition.getParameters().get(0));
    } catch (InvalidJobException e) {
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
    // Only ArtifactName and schedInfo is specified
    try {
        JobDefinition givenJobDefn = new JobDefinition.Builder().withArtifactName(artifactName).withName(clusterName).withSchedulingInfo(schedulingInfo).build();
        JobDefinitionResolver resolver = new JobDefinitionResolver();
        JobDefinition resolvedJobDefinition = resolver.getResolvedJobDefinition("user", givenJobDefn, jobClusterMetadata);
        // assert the specified values are being used
        assertEquals(artifactName, resolvedJobDefinition.getArtifactName());
        assertEquals(schedulingInfo, resolvedJobDefinition.getSchedulingInfo());
        // assert a version no was generated
        assertTrue(resolvedJobDefinition.getVersion() != null && !resolvedJobDefinition.getVersion().isEmpty());
        // assert the parameters and labels are inherited since they were not specified
        assertEquals(1, resolvedJobDefinition.getLabels().size());
        assertEquals(label, resolvedJobDefinition.getLabels().get(0));
        assertEquals(1, resolvedJobDefinition.getParameters().size());
        assertEquals(parameter, resolvedJobDefinition.getParameters().get(0));
    } catch (InvalidJobException e) {
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 13 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class LabelManager method insertSystemLabels.

static JobDefinition insertSystemLabels(JobDefinition resolvedJobDefn, boolean autoResubmit) {
    JobDefinition updatedJobDefn = resolvedJobDefn;
    if (autoResubmit) {
        updatedJobDefn = insertAutoResubmitLabel(resolvedJobDefn);
    }
    String artifactName = updatedJobDefn.getArtifactName();
    String version = updatedJobDefn.getVersion();
    List<Label> labels = updatedJobDefn.getLabels();
    // remove old artifact & version label if present.
    List<Label> updatedLabels = labels.stream().filter(label -> !(label.getName().equals(SystemLabels.MANTIS_ARTIFACT_LABEL.label))).filter(label -> !label.getName().equals(SystemLabels.MANTIS_VERSION_LABEL.label)).collect(Collectors.toList());
    updatedLabels.add(new Label(SystemLabels.MANTIS_ARTIFACT_LABEL.label, artifactName));
    updatedLabels.add(new Label(SystemLabels.MANTIS_VERSION_LABEL.label, version));
    try {
        updatedJobDefn = new JobDefinition.Builder().from(updatedJobDefn).withLabels(updatedLabels).build();
        return updatedJobDefn;
    } catch (InvalidJobException e) {
        logger.error(e.getMessage());
        return resolvedJobDefn;
    }
}

Example 14 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobClusterTest method testUpdateJobClusterArtifactWithAutoSubmit.

@Test
public void testUpdateJobClusterArtifactWithAutoSubmit() {
    TestKit probe = new TestKit(system);
    try {
        String clusterName = "testUpdateJobClusterArtifactWithAutoSubmit";
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        // when running concurrently with testGetJobDetailsForArchivedJob the following mock return is needed to avoid null pointer exception.
        when(jobStoreMock.getArchivedJob(anyString())).thenReturn(empty());
        SLA sla = new SLA(1, 1, null, null);
        final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, Lists.newArrayList(), sla);
        ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
        jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
        JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
        assertEquals(SUCCESS, createResp.responseCode);
        // submit job with different scheduling info instance count compared to cluster default one.
        final int job1InstanceCnt = 3;
        final JobDefinition jobDefn = createJob(clusterName, MantisJobDurationType.Transient, new SchedulingInfo.Builder().numberOfStages(1).addStage(fakeJobCluster.getJobClusterConfig().getSchedulingInfo().forStage(1).toBuilder().numberOfInstances(job1InstanceCnt).build()).build());
        String jobId = clusterName + "-1";
        jobClusterActor.tell(new SubmitJobRequest(clusterName, "user", Optional.ofNullable(jobDefn)), probe.getRef());
        SubmitJobResponse submitResponse = probe.expectMsgClass(SubmitJobResponse.class);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(clusterName, jobId, 0, 1));
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, BaseResponse.ResponseCode.SUCCESS, JobState.Accepted);
        // Update artifact with skip submit = false
        String artifact = "newartifact.zip";
        String version = "0.0.2";
        jobClusterActor.tell(new UpdateJobClusterArtifactRequest(clusterName, artifact, version, false, user), probe.getRef());
        UpdateJobClusterArtifactResponse resp = probe.expectMsgClass(UpdateJobClusterArtifactResponse.class);
        // ensure new job was launched
        String jobId2 = clusterName + "-2";
        assertTrue(JobTestHelper.verifyJobStatusWithPolling(probe, jobClusterActor, jobId2, JobState.Accepted));
        // send it worker events to move it to started state
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId2, 1, new WorkerId(clusterName, jobId2, 0, 1));
        jobClusterActor.tell(new GetJobDetailsRequest("nj", JobId.fromId(jobId2).get()), probe.getRef());
        GetJobDetailsResponse detailsResp = probe.expectMsgClass(Duration.ofSeconds(5), GetJobDetailsResponse.class);
        assertEquals(JobState.Accepted, detailsResp.getJobMetadata().get().getState());
        assertEquals(artifact, detailsResp.getJobMetadata().get().getArtifactName());
        // verify newly launched job inherited instance count from previous job instance.
        AtomicBoolean hasStage = new AtomicBoolean(false);
        detailsResp.getJobMetadata().get().getSchedulingInfo().getStages().forEach((stageId, stageInfo) -> {
            hasStage.set(true);
            assertEquals(job1InstanceCnt, detailsResp.getJobMetadata().get().getSchedulingInfo().forStage(stageId).getNumberOfInstances());
        });
        assertTrue(hasStage.get());
        assertTrue(JobTestHelper.verifyJobStatusWithPolling(probe, jobClusterActor, jobId2, JobState.Accepted));
    } catch (InvalidJobException e) {
        e.printStackTrace();
    }
}

Example 15 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobTestLifecycle method testListActiveWorkers.

@Test
public void testListActiveWorkers() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testListActiveWorkers";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        SchedulingInfo sInfo = new SchedulingInfo.Builder().numberOfStages(1).multiWorkerStageWithConstraints(2, new MachineDefinition(1.0, 1.0, 1.0, 3), Lists.newArrayList(), Lists.newArrayList()).build();
        jobDefn = JobTestHelper.generateJobDefinition(clusterName, sInfo);
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 2)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-2";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        int stageNo = 1;
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        // send heartbeat
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        // Only 1 worker has started.
        assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
        // send launched event
        WorkerId workerId2 = new WorkerId(jobId, 1, 2);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId2);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp3 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp3 + " msg " + resp3.message);
        assertEquals(SUCCESS, resp3.responseCode);
        // 2 worker have started so job should be started.
        assertEquals(JobState.Launched, resp3.getJobMetadata().get().getState());
        jobActor.tell(new JobClusterManagerProto.ListWorkersRequest(new JobId(clusterName, 1)), probe.getRef());
        JobClusterManagerProto.ListWorkersResponse listWorkersResponse = probe.expectMsgClass(JobClusterManagerProto.ListWorkersResponse.class);
        assertEquals(2, listWorkersResponse.getWorkerMetadata().size());
        int cnt = 0;
        for (IMantisWorkerMetadata workerMeta : listWorkersResponse.getWorkerMetadata()) {
            if (workerMeta.getWorkerNumber() == 1 || workerMeta.getWorkerNumber() == 2) {
                cnt++;
            }
        }
        assertEquals(2, cnt);
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        verify(jobStoreMock, times(6)).updateWorker(any());
        verify(jobStoreMock, times(3)).updateJob(any());
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}