Examples with MantisScheduler io.mantisrx.server.master.scheduler.MantisScheduler used on opensource projects

Example 16 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobClusterTest method testJobSubmitWithInheritInstanceFlagsScaled.

@Test
public void testJobSubmitWithInheritInstanceFlagsScaled() {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitWithInheritInstance";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    // default job with 3 stage == (2 worker)
    final SchedulingInfo schedulingInfo1 = new SchedulingInfo.Builder().numberOfStages(2).multiWorkerStage(1, DEFAULT_MACHINE_DEFINITION, true).multiWorkerStage(1, DEFAULT_MACHINE_DEFINITION, true).build();
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName, Lists.newArrayList(), NO_OP_SLA, schedulingInfo1);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    try {
        final JobDefinition jobDefn = createJob(clusterName, MantisJobDurationType.Transient, schedulingInfo1);
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 1, new WorkerId(jobId, 0, 1));
        JobTestHelper.sendLaunchedInitiatedStartedEventsToWorker(probe, jobClusterActor, jobId, 2, new WorkerId(jobId, 0, 2));
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Launched);
        // try scale job
        JobTestHelper.scaleStageAndVerify(probe, jobClusterActor, jobId, 1, 2);
        // submit another job this time with job, in which some stages with inheritInstance enabled
        final String jobId2 = clusterName + "-2";
        final SchedulingInfo schedulingInfo2 = new SchedulingInfo.Builder().numberOfStages(2).multiWorkerStage(3, DEFAULT_MACHINE_DEFINITION).multiWorkerStage(4, DEFAULT_MACHINE_DEFINITION).build();
        final DeploymentStrategy deploymentStrategy = DeploymentStrategy.builder().stage(1, StageDeploymentStrategy.builder().inheritInstanceCount(true).build()).stage(2, StageDeploymentStrategy.builder().inheritInstanceCount(true).build()).build();
        final JobDefinition jobDefn2Workers = createJob(clusterName, MantisJobDurationType.Transient, schedulingInfo2, deploymentStrategy);
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn2Workers, jobId2);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId2, SUCCESS, JobState.Accepted);
        // verify instance count is from previous job.
        final JobId jobId2Id = JobId.fromId(jobId2).get();
        jobClusterActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId2Id), probe.getRef());
        JobClusterManagerProto.GetJobDetailsResponse detailsResp = probe.expectMsgClass(Duration.ofSeconds(60), JobClusterManagerProto.GetJobDetailsResponse.class);
        assertTrue(detailsResp.getJobMetadata().isPresent());
        assertEquals(jobId2, detailsResp.getJobMetadata().get().getJobId().getId());
        final SchedulingInfo actualSchedulingInfo = detailsResp.getJobMetadata().get().getSchedulingInfo();
        assertEquals(2, actualSchedulingInfo.getStages().size());
        // stage 1 inherits from previous job while stage 2 should apply new instance count.
        assertEquals(2, actualSchedulingInfo.forStage(1).getNumberOfInstances());
        assertEquals(1, actualSchedulingInfo.forStage(2).getNumberOfInstances());
        JobTestHelper.killJobAndVerify(probe, clusterName, jobId2Id, jobClusterActor);
        verify(jobStoreMock, times(1)).createJobCluster(any());
        verify(jobStoreMock, times(2)).updateJobCluster(any());
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 17 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobClusterTest method testJobClusterSLAUpdate.

// /////////////////////////////////////////// CLUSTER CRUD END ///////////////////////////////////////////////////////
// //////////////////////////// CLUSTER UPDATE FLAVORS ///////////////////////////////////////////////////////////////
@Test
public void testJobClusterSLAUpdate() throws Exception {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobClusterSLAUpdate";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    SLA newSLA = new SLA(0, 10, null, null);
    UpdateJobClusterSLARequest updateSlaReq = new UpdateJobClusterSLARequest(clusterName, newSLA.getMin(), newSLA.getMax(), "user");
    jobClusterActor.tell(updateSlaReq, probe.getRef());
    UpdateJobClusterSLAResponse resp = probe.expectMsgClass(UpdateJobClusterSLAResponse.class);
    assertEquals(SUCCESS, resp.responseCode);
    assertEquals(jobClusterActor, probe.getLastSender());
    jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    GetJobClusterResponse resp3 = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, resp3.responseCode);
    assertTrue(resp3.getJobCluster() != null);
    System.out.println("Job cluster " + resp3.getJobCluster());
    assertEquals(clusterName, resp3.getJobCluster().get().getName());
    System.out.println("Updated job cluster " + resp3.getJobCluster());
    assertEquals(newSLA, DataFormatAdapter.convertToSLA(resp3.getJobCluster().get().getSla()));
    verify(jobStoreMock, times(1)).updateJobCluster(any());
    verify(jobStoreMock, times(1)).createJobCluster(any());
}

Example 18 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobClusterTest method testQuickJobSubmitWithNoSchedInfoInPreviousJob.

@Test
public void testQuickJobSubmitWithNoSchedInfoInPreviousJob() {
    TestKit probe = new TestKit(system);
    String clusterName = "testQuickJobSubmitWithNoSchedInfoInPreviousJob";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    try {
        // job defn with scheduling info
        final JobDefinition jobDefn = new JobDefinition.Builder().withName(clusterName).withParameters(Lists.newArrayList()).withLabels(Lists.newArrayList()).withVersion("0.0.1").withSubscriptionTimeoutSecs(300).withUser("njoshi").withJobSla(new JobSla(0, 0, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Transient, "abc")).build();
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        // submit another job this time with no job definition
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, null, clusterName + "-2");
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, clusterName + "-2", SUCCESS, JobState.Accepted);
        JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 1), jobClusterActor);
        verify(jobStoreMock, times(1)).createJobCluster(any());
        verify(jobStoreMock, times(2)).updateJobCluster(any());
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    }
// Mockito.doThrow(IOException.class).when(jobStoreMock).storeNewJob(any());
}

Example 19 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobClusterManagerTest method testBootStrapJobClustersAndJobs.

@Test
public void testBootStrapJobClustersAndJobs() {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisJobStore jobStore = new MantisJobStore(new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher));
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, false), probe.getRef());
    JobClustersManagerInitializeResponse iResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    List<String> clusterNames = Lists.newArrayList("testBootStrapJobClustersAndJobs1", "testBootStrapJobClustersAndJobs2", "testBootStrapJobClustersAndJobs3");
    String clusterWithNoJob = "testBootStrapJobClusterWithNoJob";
    createJobClusterAndAssert(jobClusterManagerActor, clusterWithNoJob);
    WorkerMigrationConfig migrationConfig = new WorkerMigrationConfig(MigrationStrategyEnum.PERCENTAGE, "{\"percentToMove\":60, \"intervalMs\":30000}");
    // Create 3 clusters and submit 1 job each
    for (String cluster : clusterNames) {
        createJobClusterAndAssert(jobClusterManagerActor, cluster, migrationConfig);
        submitJobAndAssert(jobClusterManagerActor, cluster);
        if (cluster.equals("testBootStrapJobClustersAndJobs1")) {
            // send worker events for job 1 so it goes to started state
            String jobId = "testBootStrapJobClustersAndJobs1-1";
            WorkerId workerId = new WorkerId(jobId, 0, 1);
            WorkerEvent launchedEvent = new WorkerLaunched(workerId, 0, "host1", "vm1", empty(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
            jobClusterManagerActor.tell(launchedEvent, probe.getRef());
            WorkerEvent startInitEvent = new WorkerStatus(new Status(workerId.getJobId(), 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.INFO, "test START_INIT", MantisJobState.StartInitiated));
            jobClusterManagerActor.tell(startInitEvent, probe.getRef());
            WorkerEvent heartBeat = new WorkerHeartbeat(new Status(jobId, 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.HEARTBEAT, "", MantisJobState.Started));
            jobClusterManagerActor.tell(heartBeat, probe.getRef());
            // get Job status
            jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId(jobId).get()), probe.getRef());
            GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
            // Ensure its launched
            assertEquals(SUCCESS, resp2.responseCode);
            assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        }
    }
    // kill 1 of the jobs to test archive path
    JobClusterManagerProto.KillJobRequest killRequest = new JobClusterManagerProto.KillJobRequest("testBootStrapJobClustersAndJobs2-1", JobCompletedReason.Killed.toString(), "njoshi");
    jobClusterManagerActor.tell(killRequest, probe.getRef());
    JobClusterManagerProto.KillJobResponse killJobResponse = probe.expectMsgClass(JobClusterManagerProto.KillJobResponse.class);
    assertEquals(SUCCESS, killJobResponse.responseCode);
    JobTestHelper.sendWorkerTerminatedEvent(probe, jobClusterManagerActor, "testBootStrapJobClustersAndJobs2-1", new WorkerId("testBootStrapJobClustersAndJobs2-1", 0, 1));
    try {
        Thread.sleep(500);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    // Stop job cluster Manager Actor
    system.stop(jobClusterManagerActor);
    // create new instance
    jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    // initialize it
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(JobClustersManagerInitializeResponse.class);
    // probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES),JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    // probe.expectMsgClass(JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    assertEquals(SUCCESS, initializeResponse.responseCode);
    // Get Cluster Config
    jobClusterManagerActor.tell(new GetJobClusterRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetJobClusterResponse clusterResponse = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, clusterResponse.responseCode);
    assertTrue(clusterResponse.getJobCluster().isPresent());
    WorkerMigrationConfig mConfig = clusterResponse.getJobCluster().get().getMigrationConfig();
    assertEquals(migrationConfig.getStrategy(), mConfig.getStrategy());
    assertEquals(migrationConfig.getConfigString(), migrationConfig.getConfigString());
    // get Job status
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs1-1").get()), probe.getRef());
    GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
    // Ensure its launched
    System.out.println("Resp2 -> " + resp2.message);
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
    // 1 jobs should be in completed state
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs2-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobDetailsResponse.class);
    // Ensure its completed
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Completed, resp2.getJobMetadata().get().getState());
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs3-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobDetailsResponse.class);
    // Ensure its Accepted
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
    try {
        Optional<JobWorker> workerByIndex = resp2.getJobMetadata().get().getWorkerByIndex(1, 0);
        assertTrue(workerByIndex.isPresent());
        Optional<IMantisStageMetadata> stageMetadata = resp2.getJobMetadata().get().getStageMetadata(1);
        assertTrue(stageMetadata.isPresent());
        JobWorker workerByIndex1 = stageMetadata.get().getWorkerByIndex(0);
        System.out.println("Got worker by index : " + workerByIndex1);
        Optional<JobWorker> worker = resp2.getJobMetadata().get().getWorkerByNumber(1);
        assertTrue(worker.isPresent());
    } catch (io.mantisrx.server.master.persistence.exceptions.InvalidJobException e) {
        e.printStackTrace();
    }
    jobClusterManagerActor.tell(new GetLastSubmittedJobIdStreamRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetLastSubmittedJobIdStreamResponse lastSubmittedJobIdStreamResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetLastSubmittedJobIdStreamResponse.class);
    lastSubmittedJobIdStreamResponse.getjobIdBehaviorSubject().get().take(1).toBlocking().subscribe((jId) -> {
        assertEquals(new JobId("testBootStrapJobClustersAndJobs1", 1), jId);
    });
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterWithNoJob), probe.getRef());
    GetJobClusterResponse jobClusterResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobClusterResponse.class);
    assertEquals(SUCCESS, jobClusterResponse.responseCode);
    assertTrue(jobClusterResponse.getJobCluster().isPresent());
    assertEquals(clusterWithNoJob, jobClusterResponse.getJobCluster().get().getName());
    // 1 running worker
    verify(schedulerMock, timeout(100_1000).times(1)).initializeRunningWorker(any(), any());
    // 2 worker schedule requests
    verify(schedulerMock, timeout(100_000).times(4)).scheduleWorker(any());
    try {
        Mockito.verify(jobStoreSpied).loadAllArchivedJobsAsync();
        Mockito.verify(jobStoreSpied).loadAllActiveJobs();
        Mockito.verify(jobStoreSpied).loadAllCompletedJobs();
        Mockito.verify(jobStoreSpied).archiveWorker(any());
        Mockito.verify(jobStoreSpied).archiveJob(any());
    } catch (IOException e) {
        e.printStackTrace();
        fail();
    }
}

Example 20 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobClusterManagerTest method testBootStrapJobClustersAndJobs1.

@Test
public void testBootStrapJobClustersAndJobs1() {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisJobStore jobStore = new MantisJobStore(new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher));
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse iResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    // List<String> clusterNames = Lists.newArrayList("testBootStrapJobClustersAndJobs1");
    String clusterWithNoJob = "testBootStrapJobClusterWithNoJob";
    createJobClusterAndAssert(jobClusterManagerActor, clusterWithNoJob);
    // kill 1 of the jobs to test archive path
    // Stop job cluster Manager Actor
    system.stop(jobClusterManagerActor);
    // create new instance
    jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStore, eventPublisher));
    // initialize it
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    // JobClusterManagerProto.JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    assertEquals(SUCCESS, initializeResponse.responseCode);
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterWithNoJob), probe.getRef());
    GetJobClusterResponse jobClusterResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobClusterResponse.class);
    assertEquals(SUCCESS, jobClusterResponse.responseCode);
    assertTrue(jobClusterResponse.getJobCluster().isPresent());
    assertEquals(clusterWithNoJob, jobClusterResponse.getJobCluster().get().getName());
    try {
        // Mockito.verify(jobStoreSpied).loadAllArchivedJobsAsync();
        Mockito.verify(jobStoreSpied).loadAllJobClusters();
        Mockito.verify(jobStoreSpied).loadAllActiveJobs();
        Mockito.verify(jobStoreSpied).loadAllCompletedJobs();
    // Mockito.verify(jobStoreSpied).archiveWorker(any());
    // Mockito.verify(jobStoreSpied).archiveJob(any());
    } catch (IOException e) {
        e.printStackTrace();
        fail();
    }
}