Examples with InvalidJobException io.mantisrx.runtime.command.InvalidJobException used on opensource projects

Example 16 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobTestLifecycle method testJobSubmit.

@Test
public void testJobSubmit() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitCluster";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        jobDefn = JobTestHelper.generateJobDefinition(clusterName);
        // IMantisStorageProvider storageProvider = new SimpleCachedFileStorageProvider();
        // MantisJobStore	jobStore = new MantisJobStore(storageProvider);
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 1)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-1";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        assertTrue(resp.getJobMetadata().get().getStageMetadata(1).isPresent());
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        int stageNum = 1;
        JobTestHelper.sendWorkerLaunchedEvent(probe, jobActor, workerId, stageNum);
        JobTestHelper.sendStartInitiatedEvent(probe, jobActor, stageNum, workerId);
        // send heartbeat
        JobTestHelper.sendHeartBeat(probe, jobActor, jobId, stageNum, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        verify(jobStoreMock, times(3)).updateWorker(any());
        verify(jobStoreMock, times(3)).updateJob(any());
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 17 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobTestLifecycle method testJobSubmitWithMultipleStagesAndWorkers.

@Test
public void testJobSubmitWithMultipleStagesAndWorkers() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitWithMultipleStagesAndWorkers";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        Map<StageScalingPolicy.ScalingReason, StageScalingPolicy.Strategy> smap = new HashMap<>();
        smap.put(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, 0.1, 0.6, null));
        SchedulingInfo.Builder builder = new SchedulingInfo.Builder().numberOfStages(2).multiWorkerScalableStageWithConstraints(2, new MachineDefinition(1, 1.24, 0.0, 1, 1), null, null, new StageScalingPolicy(1, 1, 3, 1, 1, 60, smap)).multiWorkerScalableStageWithConstraints(3, new MachineDefinition(1, 1.24, 0.0, 1, 1), null, null, new StageScalingPolicy(1, 1, 3, 1, 1, 60, smap));
        SchedulingInfo sInfo = builder.build();
        System.out.println("SchedulingInfo " + sInfo);
        jobDefn = JobTestHelper.generateJobDefinition(clusterName, sInfo);
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 1)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-1";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        int stageNo = 0;
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        // send heartbeat
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        // Only 1 worker has started.
        assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
        // send launched events for the rest of the workers
        int nextWorkerNumber = 1;
        int stage = 0;
        Iterator<Map.Entry<Integer, StageSchedulingInfo>> it = sInfo.getStages().entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<Integer, StageSchedulingInfo> integerStageSchedulingInfoEntry = it.next();
            StageSchedulingInfo stageSchedulingInfo = integerStageSchedulingInfoEntry.getValue();
            System.out.println("Workers -> " + stageSchedulingInfo.getNumberOfInstances() + " in stage " + stage);
            for (int i = 0; i < stageSchedulingInfo.getNumberOfInstances(); i++) {
                WorkerId wId = new WorkerId(jobId, i, nextWorkerNumber++);
                System.out.println("Sending events for worker --> " + wId + " Stage " + stage);
                JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stage, wId);
            }
            stage++;
        }
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp3 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp3 + " msg " + resp3.message);
        assertEquals(SUCCESS, resp3.responseCode);
        // 2 worker have started so job should be started.
        assertEquals(JobState.Launched, resp3.getJobMetadata().get().getState());
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        verify(jobStoreMock, times(19)).updateWorker(any());
        verify(jobStoreMock, times(3)).updateJob(any());
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 18 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobTestLifecycle method testHeartBeatEnforcement.

@Test
public void testHeartBeatEnforcement() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testHeartBeatEnforcementCluster";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        SchedulingInfo sInfo = new SchedulingInfo.Builder().numberOfStages(1).multiWorkerStageWithConstraints(2, new MachineDefinition(1.0, 1.0, 1.0, 3), Lists.newArrayList(), Lists.newArrayList()).build();
        jobDefn = JobTestHelper.generateJobDefinition(clusterName, sInfo);
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 2)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-2";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        int stageNo = 1;
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        // send Launched, Initiated and heartbeat
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        // Only 1 worker has started.
        assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
        // send launched event
        WorkerId workerId2 = new WorkerId(jobId, 1, 2);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId2);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp3 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp3 + " msg " + resp3.message);
        assertEquals(SUCCESS, resp3.responseCode);
        // 2 worker have started so job should be started.
        assertEquals(JobState.Launched, resp3.getJobMetadata().get().getState());
        JobTestHelper.sendHeartBeat(probe, jobActor, jobId, 1, workerId2);
        JobTestHelper.sendHeartBeat(probe, jobActor, jobId, 1, workerId);
        // check hb status in the future where we expect all last HBs to be stale.
        Instant now = Instant.now();
        jobActor.tell(new JobProto.CheckHeartBeat(now.plusSeconds(240)), probe.getRef());
        Thread.sleep(1000);
        // 2 original submissions and 2 resubmits because of HB timeouts
        verify(schedulerMock, times(4)).scheduleWorker(any());
        // 2 kills due to resubmits
        verify(schedulerMock, times(2)).unscheduleAndTerminateWorker(any(), any());
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 19 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobTestLifecycle method testJobSubmitPerpetual.

@Test
public void testJobSubmitPerpetual() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitPerpetual";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        MachineDefinition machineDefinition = new MachineDefinition(1.0, 1.0, 1.0, 1.0, 3);
        SchedulingInfo schedInfo = new SchedulingInfo.Builder().numberOfStages(1).singleWorkerStageWithConstraints(machineDefinition, Lists.newArrayList(), Lists.newArrayList()).build();
        jobDefn = new JobDefinition.Builder().withName(clusterName).withParameters(Lists.newArrayList()).withLabels(Lists.newArrayList()).withSchedulingInfo(schedInfo).withArtifactName("myart").withSubscriptionTimeoutSecs(30).withUser("njoshi").withNumberOfStages(schedInfo.getStages().size()).withJobSla(new JobSla(0, 0, null, MantisJobDurationType.Perpetual, null)).build();
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 1)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-1";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        assertTrue(resp.getJobMetadata().get().getStageMetadata(1).isPresent());
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        int stageNum = 1;
        JobTestHelper.sendWorkerLaunchedEvent(probe, jobActor, workerId, stageNum);
        JobTestHelper.sendStartInitiatedEvent(probe, jobActor, stageNum, workerId);
        // send heartbeat
        JobTestHelper.sendHeartBeat(probe, jobActor, jobId, stageNum, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        verify(jobStoreMock, times(3)).updateWorker(any());
        verify(jobStoreMock, times(3)).updateJob(any());
        // verify(jobStoreMock, times(3))
        verify(schedulerMock, times(1)).scheduleWorker(any());
        JobMetadata jobMetadata = new JobMetadata(jobId, new URL("http://myart" + ""), 1, "njoshi", schedInfo, Lists.newArrayList(), 0, 0);
        ScheduleRequest expectedScheduleRequest = new ScheduleRequest(workerId, 1, 4, jobMetadata, MantisJobDurationType.Perpetual, machineDefinition, Lists.newArrayList(), Lists.newArrayList(), 0, empty());
        verify(schedulerMock).scheduleWorker(expectedScheduleRequest);
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 20 with InvalidJobException

use of io.mantisrx.runtime.command.InvalidJobException in project mantis by Netflix.

the class JobTestLifecycle method testJobSubmitWithMultipleWorkers.

@Test
public void testJobSubmitWithMultipleWorkers() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitWithMultipleWorkersCluster";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        SchedulingInfo sInfo = new SchedulingInfo.Builder().numberOfStages(1).multiWorkerStageWithConstraints(2, new MachineDefinition(1.0, 1.0, 1.0, 3), Lists.newArrayList(), Lists.newArrayList()).build();
        jobDefn = JobTestHelper.generateJobDefinition(clusterName, sInfo);
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 2)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-2";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        int stageNo = 1;
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        // send heartbeat
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        // Only 1 worker has started.
        assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
        // send launched event
        WorkerId workerId2 = new WorkerId(jobId, 1, 2);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobActor, jobId, stageNo, workerId2);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp3 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp3 + " msg " + resp3.message);
        assertEquals(SUCCESS, resp3.responseCode);
        // 2 worker have started so job should be started.
        assertEquals(JobState.Launched, resp3.getJobMetadata().get().getState());
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        verify(jobStoreMock, times(6)).updateWorker(any());
        verify(jobStoreMock, times(3)).updateJob(any());
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}