Examples with DisableJobClusterRequest io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest used on opensource projects

Example 1 with DisableJobClusterRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest in project mantis by Netflix.

the class JobClusterTest method testJobClusterDisable.

@Test
public void testJobClusterDisable() throws InterruptedException {
    TestKit probe = new TestKit(system);
    CountDownLatch storeCompletedCalled = new CountDownLatch(1);
    String clusterName = "testJobClusterDisable";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    try {
        final JobDefinition jobDefn = createJob(clusterName, 1, MantisJobDurationType.Transient);
        String jobId = clusterName + "-1";
        IMantisJobMetadata completedJobMock = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 1)).withJobDefinition(jobDefn).withJobState(JobState.Completed).build();
        when(jobStoreMock.getArchivedJob(any())).thenReturn(of(completedJobMock));
        doAnswer((Answer) invocation -> {
            storeCompletedCalled.countDown();
            return null;
        }).when(jobStoreMock).storeCompletedJobForCluster(any(), any());
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        jobClusterActor.tell(new DisableJobClusterRequest(clusterName, "user"), probe.getRef());
        DisableJobClusterResponse disableResp = probe.expectMsgClass(DisableJobClusterResponse.class);
        assertEquals(SUCCESS, disableResp.responseCode);
        jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
        GetJobClusterResponse getJobClusterResp = probe.expectMsgClass(GetJobClusterResponse.class);
        assertTrue(getJobClusterResp.getJobCluster().get().isDisabled());
        jobClusterActor.tell(new GetJobDetailsRequest(clusterName, JobId.fromId(jobId).get()), probe.getRef());
        GetJobDetailsResponse jobDetailsResp = probe.expectMsgClass(GetJobDetailsResponse.class);
        assertEquals(SUCCESS, jobDetailsResp.responseCode);
        assertEquals(jobId, jobDetailsResp.getJobMetadata().get().getJobId().getId());
        assertEquals(JobState.Completed, jobDetailsResp.getJobMetadata().get().getState());
        verify(jobStoreMock, times(1)).createJobCluster(any());
        verify(jobStoreMock, times(2)).updateJobCluster(any());
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).updateStage(any());
        verify(jobStoreMock, times(2)).updateJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        storeCompletedCalled.await(1, TimeUnit.SECONDS);
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 2 with DisableJobClusterRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest in project mantis by Netflix.

the class JobClusterManagerTest method testJobClusterEnable.

@Test
public void testJobClusterEnable() throws MalformedURLException {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobClusterEnable";
    List<Label> labels = Lists.newLinkedList();
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, labels);
    jobClusterManagerActor.tell(new JobClusterManagerProto.CreateJobClusterRequest(fakeJobCluster, "user"), probe.getRef());
    JobClusterManagerProto.CreateJobClusterResponse createResp = probe.expectMsgClass(JobClusterManagerProto.CreateJobClusterResponse.class);
    assertEquals(SUCCESS_CREATED, createResp.responseCode);
    DisableJobClusterRequest req = new JobClusterManagerProto.DisableJobClusterRequest(clusterName, "user");
    jobClusterManagerActor.tell(req, probe.getRef());
    JobClusterManagerProto.DisableJobClusterResponse updateResp = probe.expectMsgClass(JobClusterManagerProto.DisableJobClusterResponse.class);
    assertEquals(SUCCESS, updateResp.responseCode);
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    GetJobClusterResponse getResp = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, getResp.responseCode);
    assertTrue(getResp.getJobCluster().get().isDisabled());
    EnableJobClusterRequest req2 = new JobClusterManagerProto.EnableJobClusterRequest(clusterName, "user");
    jobClusterManagerActor.tell(req2, probe.getRef());
    JobClusterManagerProto.EnableJobClusterResponse updateResp2 = probe.expectMsgClass(JobClusterManagerProto.EnableJobClusterResponse.class);
    assertEquals(SUCCESS, updateResp2.responseCode);
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    getResp = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, getResp.responseCode);
    assertFalse(getResp.getJobCluster().get().isDisabled());
}

Example 3 with DisableJobClusterRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest in project mantis by Netflix.

the class JobClusterTest method testJobClusterDeletePurgesCompletedJobs.

@Test
public void testJobClusterDeletePurgesCompletedJobs() throws Exception {
    TestKit probe = new TestKit(system);
    List<Label> labels = Lists.newLinkedList();
    Label l = new Label("labelname", "labelvalue");
    labels.add(l);
    String clusterName = "testJobClusterDeletePurgesCompletedJobs";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, labels);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    final JobDefinition jobDefn = createJob(clusterName, 1, MantisJobDurationType.Transient);
    String jobId = clusterName + "-1";
    JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
    JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
    jobClusterActor.tell(new DisableJobClusterRequest(clusterName, "user"), probe.getRef());
    DisableJobClusterResponse disableResp = probe.expectMsgClass(DisableJobClusterResponse.class);
    assertEquals(SUCCESS, disableResp.responseCode);
    Thread.sleep(1000);
    jobClusterActor.tell(new JobClusterProto.DeleteJobClusterRequest(user, clusterName, probe.getRef()), probe.getRef());
    JobClusterProto.DeleteJobClusterResponse resp4 = probe.expectMsgClass(JobClusterProto.DeleteJobClusterResponse.class);
    assertEquals(SUCCESS, resp4.responseCode);
    assertEquals(jobClusterActor, probe.getLastSender());
    verify(jobStoreMock, times(1)).createJobCluster(any());
    verify(jobStoreMock, times(2)).updateJobCluster(any());
    verify(jobStoreMock, times(1)).deleteJobCluster(clusterName);
    verify(jobStoreMock, times(1)).storeCompletedJobForCluster(any(), any());
    verify(jobStoreMock, times(1)).deleteJob("testJobClusterDeletePurgesCompletedJobs-1");
}

Example 4 with DisableJobClusterRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest in project mantis by Netflix.

the class JobClusterRoute method getJobClusterRoutes.

private Route getJobClusterRoutes() {
    return route(path(segment("api").slash("submit"), () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/submit called {}", request);
        try {
            MantisJobDefinition mjd = Jackson.fromJSON(request, MantisJobDefinition.class);
            logger.debug("job submit request {}", mjd);
            mjd.validate(true);
            Pair<Boolean, String> validationResult = validateSubmitJobRequest(mjd);
            if (!validationResult.first()) {
                jobClusterSubmitError.increment();
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"" + validationResult.second() + "\"}");
            }
            jobClusterSubmit.increment();
            return completeWithFuture(jobClusterRouteHandler.submit(JobClusterProtoAdapter.toSubmitJobClusterRequest(mjd)).thenApply(this::toHttpResponse));
        } catch (Exception e) {
            logger.warn("exception in submit job request {}", request, e);
            jobClusterSubmitError.increment();
            return complete(StatusCodes.INTERNAL_SERVER_ERROR, "{\"error\": \"" + e.getMessage() + "\"}");
        }
    }))), pathPrefix(API_V0_JOBCLUSTER, () -> route(post(() -> route(path("create", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), jobClusterDefn -> {
        logger.debug("/api/namedjob/create called {}", jobClusterDefn);
        try {
            final NamedJobDefinition namedJobDefinition = Jackson.fromJSON(jobClusterDefn, NamedJobDefinition.class);
            if (namedJobDefinition == null || namedJobDefinition.getJobDefinition() == null || namedJobDefinition.getJobDefinition().getJobJarFileLocation() == null || namedJobDefinition.getJobDefinition().getName() == null || namedJobDefinition.getJobDefinition().getName().isEmpty()) {
                logger.warn("JobCluster create request must include name and URL {}", jobClusterDefn);
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"Job definition must include name and URL\"}");
            }
            final CompletionStage<CreateJobClusterResponse> response = jobClusterRouteHandler.create(JobClusterProtoAdapter.toCreateJobClusterRequest(namedJobDefinition));
            jobClusterCreate.increment();
            return completeWithFuture(response.thenApply(r -> {
                if ((r.responseCode == CLIENT_ERROR || r.responseCode == CLIENT_ERROR_CONFLICT) && r.message.contains("already exists")) {
                    return new CreateJobClusterResponse(r.requestId, SERVER_ERROR, r.message, r.getJobClusterName());
                }
                return r;
            }).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error creating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't read valid json in request: " + e.getMessage());
        } catch (Exception e) {
            logger.warn("Error creating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateError.increment();
            return complete(StatusCodes.INTERNAL_SERVER_ERROR, "{\"error\": " + e.getMessage() + "}");
        }
    }))), path("update", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), jobClusterDefn -> {
        logger.debug("/api/namedjob/update called {}", jobClusterDefn);
        try {
            final NamedJobDefinition namedJobDefinition = Jackson.fromJSON(jobClusterDefn, NamedJobDefinition.class);
            if (namedJobDefinition == null || namedJobDefinition.getJobDefinition() == null || namedJobDefinition.getJobDefinition().getJobJarFileLocation() == null || namedJobDefinition.getJobDefinition().getName() == null || namedJobDefinition.getJobDefinition().getName().isEmpty()) {
                logger.warn("JobCluster update request must include name and URL {}", jobClusterDefn);
                jobClusterCreateUpdateError.increment();
                return complete(StatusCodes.BAD_REQUEST, "{\"error\": \"Job definition must include name and URL\"}");
            }
            final CompletionStage<UpdateJobClusterResponse> response = jobClusterRouteHandler.update(JobClusterProtoAdapter.toUpdateJobClusterRequest(namedJobDefinition));
            jobClusterCreateUpdate.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateUpdateError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't read valid json in request: " + e.getMessage());
        } catch (Exception e) {
            logger.warn("Error updating JobCluster {}", jobClusterDefn, e);
            jobClusterCreateUpdateError.increment();
            return complete(StatusCodes.INTERNAL_SERVER_ERROR, "{\"error\": " + e.getMessage() + "}");
        }
    }))), path("delete", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), deleteReq -> {
        logger.debug("/api/namedjob/delete called {}", deleteReq);
        try {
            final DeleteJobClusterRequest deleteJobClusterRequest = Jackson.fromJSON(deleteReq, DeleteJobClusterRequest.class);
            final CompletionStage<DeleteJobClusterResponse> response = jobClusterRouteHandler.delete(deleteJobClusterRequest);
            jobClusterDelete.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error deleting JobCluster {}", deleteReq, e);
            jobClusterDeleteError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("disable", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/disable called {}", request);
        try {
            final DisableJobClusterRequest disableJobClusterRequest = Jackson.fromJSON(request, DisableJobClusterRequest.class);
            final CompletionStage<DisableJobClusterResponse> response = jobClusterRouteHandler.disable(disableJobClusterRequest);
            jobClusterDisable.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error disabling JobCluster {}", request, e);
            jobClusterDisableError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("enable", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/enable called {}", request);
        try {
            final EnableJobClusterRequest enableJobClusterRequest = Jackson.fromJSON(request, EnableJobClusterRequest.class);
            final CompletionStage<EnableJobClusterResponse> response = jobClusterRouteHandler.enable(enableJobClusterRequest);
            jobClusterEnable.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error enabling JobCluster {}", request, e);
            jobClusterEnableError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("quickupdate", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/quickupdate called {}", request);
        try {
            final UpdateJobClusterArtifactRequest updateJobClusterArtifactRequest = Jackson.fromJSON(request, UpdateJobClusterArtifactRequest.class);
            final CompletionStage<UpdateJobClusterArtifactResponse> response = jobClusterRouteHandler.updateArtifact(updateJobClusterArtifactRequest);
            jobClusterQuickupdate.increment();
            return completeWithFuture(response.thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on quickupdate for JobCluster {}", request, e);
            jobClusterQuickupdateError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("updatelabels", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/updatelabels called {}", request);
        try {
            final UpdateJobClusterLabelsRequest updateJobClusterLabelsRequest = Jackson.fromJSON(request, UpdateJobClusterLabelsRequest.class);
            jobClusterUpdateLabel.increment();
            return completeWithFuture(jobClusterRouteHandler.updateLabels(updateJobClusterLabelsRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating labels for JobCluster {}", request, e);
            jobClusterUpdateLabelError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("updatesla", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/updatesla called {}", request);
        jobClusterUpdateSla.increment();
        try {
            final UpdateJobClusterSLARequest updateJobClusterSLARequest = Jackson.fromJSON(request, UpdateJobClusterSLARequest.class);
            return completeWithFuture(jobClusterRouteHandler.updateSLA(updateJobClusterSLARequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating SLA for JobCluster {}", request, e);
            jobClusterUpdateSlaError.increment();
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("migratestrategy", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/migratestrategy called {}", request);
        try {
            final UpdateJobClusterWorkerMigrationStrategyRequest updateMigrateStrategyReq = Jackson.fromJSON(request, UpdateJobClusterWorkerMigrationStrategyRequest.class);
            return completeWithFuture(jobClusterRouteHandler.updateWorkerMigrateStrategy(updateMigrateStrategyReq).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error updating migrate strategy for JobCluster {}", request, e);
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))), path("quicksubmit", () -> decodeRequest(() -> entity(Unmarshaller.entityToString(), request -> {
        logger.debug("/api/namedjob/quicksubmit called {}", request);
        try {
            final JobClusterManagerProto.SubmitJobRequest submitJobRequest = Jackson.fromJSON(request, JobClusterManagerProto.SubmitJobRequest.class);
            return completeWithFuture(jobClusterRouteHandler.submit(submitJobRequest).thenApply(this::toHttpResponse));
        } catch (IOException e) {
            logger.warn("Error on quick submit for JobCluster {}", request, e);
            return complete(StatusCodes.BAD_REQUEST, "Can't find valid json in request: " + e.getMessage());
        }
    }))))), get(() -> route(pathPrefix("list", () -> route(pathEndOrSingleSlash(() -> {
        logger.debug("/api/namedjob/list called");
        jobClusterListGET.increment();
        return alwaysCache(cache, requestUriKeyer, () -> extractUri(uri -> completeAsync(jobClusterRouteHandler.getAllJobClusters(new ListJobClustersRequest()), resp -> completeOK(resp.getJobClusters().stream().map(jobClusterMetadataView -> JobClusterProtoAdapter.toJobClusterInfo(jobClusterMetadataView)).collect(Collectors.toList()), Jackson.marshaller()), resp -> completeOK(Collections.emptyList(), Jackson.marshaller()))));
    }), path(PathMatchers.segment(), (jobCluster) -> {
        if (logger.isDebugEnabled()) {
            logger.debug("/api/namedjob/list/{} called", jobCluster);
        }
        jobClusterListClusterGET.increment();
        return completeAsync(jobClusterRouteHandler.getJobClusterDetails(new JobClusterManagerProto.GetJobClusterRequest(jobCluster)), resp -> completeOK(resp.getJobCluster().map(jc -> Arrays.asList(jc)).orElse(Collections.emptyList()), Jackson.marshaller()), resp -> completeOK(Collections.emptyList(), Jackson.marshaller()));
    }))), path(segment("listJobIds").slash(PathMatchers.segment()), (jobCluster) -> {
        logger.debug("/api/namedjob/listJobIds/{} called", jobCluster);
        jobClusterListJobIdGET.increment();
        return jobClusterListRoute(jobCluster);
    }), path("listJobIds", () -> {
        logger.debug("/api/namedjob/listJobIds called");
        return complete(StatusCodes.BAD_REQUEST, "Specify the Job cluster name '/api/namedjob/listJobIds/<JobClusterName>' to list the job Ids");
    }))))));
}

Example 5 with DisableJobClusterRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.DisableJobClusterRequest in project mantis by Netflix.

the class JobClusterTest method testJobClusterEnable.

@Test
public void testJobClusterEnable() {
    try {
        TestKit probe = new TestKit(system);
        String clusterName = "testJobClusterEnable";
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        String jobId = clusterName + "-1";
        JobDefinition jobDefn = createJob(clusterName);
        IMantisJobMetadata job1 = new MantisJobMetadataImpl.Builder().withJobDefinition(jobDefn).withJobState(JobState.Completed).withJobId(new JobId(clusterName, 1)).withNextWorkerNumToUse(2).withSubmittedAt(1000).build();
        when(jobStoreMock.getArchivedJob(jobId)).thenReturn(of(job1));
        SLA sla = new SLA(1, 1, null, null);
        final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, Lists.newArrayList(), sla);
        ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
        jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
        JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
        assertEquals(SUCCESS, createResp.responseCode);
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(clusterName, jobId, 0, 1));
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Launched);
        jobClusterActor.tell(new DisableJobClusterRequest(clusterName, user), probe.getRef());
        DisableJobClusterResponse resp = probe.expectMsgClass(DisableJobClusterResponse.class);
        assertTrue(BaseResponse.ResponseCode.SUCCESS.equals(resp.responseCode));
        jobClusterActor.tell(new EnableJobClusterRequest(clusterName, user), probe.getRef());
        EnableJobClusterResponse enableResp = probe.expectMsgClass(EnableJobClusterResponse.class);
        assertTrue(BaseResponse.ResponseCode.SUCCESS.equals(enableResp.responseCode));
        // first job was killed during disable
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Completed);
        // Sla will cause new job to get launched
        String jobId2 = clusterName + "-2";
        boolean accepted = false;
        int cnt = 0;
        // try a few times for timing issue
        while (cnt < 50) {
            jobClusterActor.tell(new GetJobDetailsRequest("nj", JobId.fromId(jobId2).get()), probe.getRef());
            GetJobDetailsResponse detailsResp = probe.expectMsgClass(GetJobDetailsResponse.class);
            if (detailsResp.responseCode.equals(BaseResponse.ResponseCode.SUCCESS)) {
                accepted = true;
                break;
            }
            Thread.sleep(1000);
        }
        assertTrue(accepted);
    // JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Completed);
    // JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 2), jobClusterActor);
    // verify(jobStoreMock, times(1)).createJobCluster(any());
    // verify(jobStoreMock, times(1)).updateJobCluster(any());
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    }
// Mockito.doThrow(IOException.class).when(jobStoreMock).storeNewJob(any());
}