Examples with ListJobsRequest io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsRequest used on opensource projects

Example 1 with ListJobsRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsRequest in project mantis by Netflix.

the class JobClusterActor method onJobList.

@Override
public void onJobList(final ListJobsRequest request) {
    if (logger.isDebugEnabled()) {
        logger.info("Entering JCA:onJobList");
    }
    final ActorRef sender = getSender();
    final ActorRef self = getSelf();
    Set<JobId> jobIdsFilteredByLabelsSet = new HashSet<>();
    // If labels criterion is given prefilter by labels
    if (!request.getCriteria().getMatchingLabels().isEmpty()) {
        jobIdsFilteredByLabelsSet = jobManager.getJobsMatchingLabels(request.getCriteria().getMatchingLabels(), request.getCriteria().getLabelsOperand());
        // Found no jobs matching labels exit
        if (jobIdsFilteredByLabelsSet.isEmpty()) {
            if (logger.isTraceEnabled()) {
                logger.trace("Exit JCA:onJobList {}", jobIdsFilteredByLabelsSet.size());
            }
            sender.tell(new ListJobsResponse(request.requestId, SUCCESS, "", new ArrayList<>()), self);
            return;
        }
    }
    // Found jobs matching labels or no labels criterion given.
    // Apply additional criterion to both active and completed jobs
    getFilteredNonTerminalJobList(request.getCriteria(), jobIdsFilteredByLabelsSet).mergeWith(getFilteredTerminalJobList(request.getCriteria(), jobIdsFilteredByLabelsSet)).collect(() -> Lists.<MantisJobMetadataView>newArrayList(), List::add).doOnNext(resultList -> {
        if (logger.isTraceEnabled()) {
            logger.trace("Exit JCA:onJobList {}", resultList.size());
        }
        sender.tell(new ListJobsResponse(request.requestId, SUCCESS, "", resultList), self);
    }).subscribe();
}

Example 2 with ListJobsRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsRequest in project mantis by Netflix.

the class JobClusterTest method testLostWorkerGetsReplaced.

@Test
public void testLostWorkerGetsReplaced() {
    TestKit probe = new TestKit(system);
    String clusterName = "testLostWorkerGetsReplaced";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    // MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreSpied, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    try {
        final JobDefinition jobDefn = createJob(clusterName, 1, MantisJobDurationType.Transient);
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        // JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        // JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 1), jobClusterActor);
        verify(jobStoreSpied, times(1)).createJobCluster(any());
        verify(jobStoreSpied, times(1)).updateJobCluster(any());
        int stageNo = 1;
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        // send heartbeat
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, stageNo, workerId);
        // check job status again
        jobClusterActor.tell(new GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        // Job started
        assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        // send launched event
        // worker 2 gets terminated abnormally
        JobTestHelper.sendWorkerTerminatedEvent(probe, jobClusterActor, jobId, workerId);
        // replaced worker comes up and sends events
        WorkerId workerId2_replaced = new WorkerId(jobId, 0, 2);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, stageNo, workerId2_replaced);
        jobClusterActor.tell(new GetJobDetailsRequest("nj", jobId), probe.getRef());
        GetJobDetailsResponse resp4 = probe.expectMsgClass(GetJobDetailsResponse.class);
        IMantisJobMetadata jobMeta = resp4.getJobMetadata().get();
        Map<Integer, ? extends IMantisStageMetadata> stageMetadata = jobMeta.getStageMetadata();
        IMantisStageMetadata stage = stageMetadata.get(1);
        for (JobWorker worker : stage.getAllWorkers()) {
            System.out.println("worker -> " + worker.getMetadata());
        }
        // 2 initial schedules and 1 replacement
        verify(schedulerMock, timeout(1_000).times(2)).scheduleWorker(any());
        // archive worker should get called once for the dead worker
        // verify(jobStoreMock, timeout(1_000).times(1)).archiveWorker(any());
        Mockito.verify(jobStoreSpied).archiveWorker(any());
        jobClusterActor.tell(new ListJobsRequest(), probe.getRef());
        ListJobsResponse listResp2 = probe.expectMsgClass(ListJobsResponse.class);
        assertEquals(SUCCESS, listResp2.responseCode);
        assertEquals(1, listResp2.getJobList().size());
        for (MantisJobMetadataView jb : listResp2.getJobList()) {
            System.out.println("Jb -> " + jb);
        }
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    } finally {
        system.stop(jobClusterActor);
    }
}

Example 3 with ListJobsRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsRequest in project mantis by Netflix.

the class JobClusterTest method testListJobWithLabelMatch.

@Test
public void testListJobWithLabelMatch() {
    TestKit probe = new TestKit(system);
    String clusterName = "testListJobWithLabelMatch";
    try {
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
        ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
        jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
        JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
        assertEquals(SUCCESS, createResp.responseCode);
        final JobDefinition jobDefn1;
        List<Label> labelList1 = new ArrayList<>();
        labelList1.add(new Label("l1", "l1v1"));
        jobDefn1 = createJob(clusterName, labelList1);
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn1, jobId);
        List<Label> labelList2 = new ArrayList<>();
        labelList2.add(new Label("l2", "l2v2"));
        String jobId2 = clusterName + "-2";
        JobDefinition jobDefn2 = createJob(clusterName, labelList2);
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn2, jobId2);
        // Query for Label1
        List<Integer> emptyIntList = Lists.newArrayList();
        List<WorkerState.MetaState> workerState = Lists.newArrayList();
        ListJobCriteria criteria1 = new ListJobCriteria(Optional.empty(), Optional.empty(), emptyIntList, emptyIntList, emptyIntList, workerState, Optional.empty(), Optional.empty(), of("l1=l1v1"), Optional.empty());
        jobClusterActor.tell(new ListJobsRequest(criteria1), probe.getRef());
        ListJobsResponse listResp = probe.expectMsgClass(ListJobsResponse.class);
        assertEquals(SUCCESS, listResp.responseCode);
        // Only job1 should be returned
        assertEquals(1, listResp.getJobList().size());
        assertEquals(jobId, listResp.getJobList().get(0).getJobMetadata().getJobId());
        assertTrue(listResp.getJobList().get(0).getStageMetadataList().size() == 1);
        System.out.println("Workers returned : " + listResp.getJobList().get(0).getWorkerMetadataList());
        assertTrue(listResp.getJobList().get(0).getWorkerMetadataList().size() == 1);
        // Query with an OR query for both labels
        ListJobCriteria criteria2 = new ListJobCriteria(Optional.empty(), Optional.empty(), emptyIntList, emptyIntList, emptyIntList, workerState, Optional.empty(), Optional.empty(), of("l1=l1v1,l2=l2v2"), Optional.empty());
        jobClusterActor.tell(new ListJobsRequest(criteria2), probe.getRef());
        ListJobsResponse listRes2 = probe.expectMsgClass(ListJobsResponse.class);
        assertEquals(SUCCESS, listRes2.responseCode);
        // Both jobs should be returned
        assertEquals(2, listRes2.getJobList().size());
        assertTrue(jobId.equals(listRes2.getJobList().get(0).getJobMetadata().getJobId()) || jobId.equals(listRes2.getJobList().get(1).getJobMetadata().getJobId()));
        assertTrue(jobId2.equals(listRes2.getJobList().get(0).getJobMetadata().getJobId()) || jobId2.equals(listRes2.getJobList().get(1).getJobMetadata().getJobId()));
        // Query with an AND query for both labels
        ListJobCriteria criteria3 = new ListJobCriteria(Optional.empty(), Optional.empty(), emptyIntList, emptyIntList, emptyIntList, workerState, Optional.empty(), Optional.empty(), of("l1=l1v1,l2=l2v2"), of("and"));
        jobClusterActor.tell(new ListJobsRequest(criteria3), probe.getRef());
        ListJobsResponse listRes3 = probe.expectMsgClass(ListJobsResponse.class);
        assertEquals(SUCCESS, listRes3.responseCode);
        // No jobs should be returned
        assertEquals(0, listRes3.getJobList().size());
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 4 with ListJobsRequest

use of io.mantisrx.master.jobcluster.proto.JobClusterManagerProto.ListJobsRequest in project mantis by Netflix.

the class JobClusterTest method testListJobsForCluster.

@Test
public void testListJobsForCluster() throws InvalidJobException, InterruptedException {
    TestKit probe = new TestKit(system);
    String clusterName = "testListJobsForCluster";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    final JobDefinition jobDefn1 = createJob(clusterName);
    String jobId = clusterName + "-1";
    JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn1, jobId);
    JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(clusterName, jobId, 0, 1));
    String jobId2 = clusterName + "-2";
    JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn1, jobId2);
    JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId2, 1, new WorkerId(clusterName, jobId2, 0, 1));
    jobClusterActor.tell(new ListJobsRequest(), probe.getRef());
    // Thread.sleep(1000);
    ListJobsResponse listResp = probe.expectMsgClass(ListJobsResponse.class);
    assertEquals(SUCCESS, listResp.responseCode);
    assertEquals(2, listResp.getJobList().size());
    // assertTrue(listResp.getJobIds().contains(JobId.fromId(jobId).get()));
    // assertTrue(listResp.getJobIds().contains(JobId.fromId(jobId2).get()));
    JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 1), jobClusterActor);
    jobClusterActor.tell(new ListJobsRequest(new ListJobCriteria(empty(), empty(), Lists.newArrayList(), Lists.newArrayList(), Lists.newArrayList(), Lists.newArrayList(), of(true), empty(), empty(), empty())), probe.getRef());
    ListJobsResponse listResp2 = probe.expectMsgClass(ListJobsResponse.class);
    assertEquals(SUCCESS, listResp2.responseCode);
    assertEquals(1, listResp2.getJobList().size());
    // assertFalse(listResp2.getJobIds().contains(JobId.fromId(jobId).get()));
    // assertTrue(listResp2.getJobIds().contains(JobId.fromId(jobId2).get()));
    JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 2), jobClusterActor);
    jobClusterActor.tell(new ListJobsRequest(new ListJobCriteria(empty(), empty(), Lists.newArrayList(), Lists.newArrayList(), Lists.newArrayList(), Lists.newArrayList(), of(true), empty(), empty(), empty())), probe.getRef());
    ListJobsResponse listResp3 = probe.expectMsgClass(ListJobsResponse.class);
    assertEquals(SUCCESS, listResp3.responseCode);
    assertEquals(0, listResp3.getJobList().size());
// assertFalse(listResp3.getJobIds().contains(JobId.fromId(jobId).get()));
// assertFalse(listResp3.getJobIds().contains(JobId.fromId(jobId2).get()));
}