Examples with WorkerLaunched io.mantisrx.server.master.scheduler.WorkerLaunched used on opensource projects

Example 1 with WorkerLaunched

use of io.mantisrx.server.master.scheduler.WorkerLaunched in project mantis by Netflix.

the class SchedulingService method launchTasks.

/**
 * Attempts to launch tasks given some number of leases from Mesos.
 *
 * When a task is launched successfully, the following will happen:
 *
 * 1. Emit a {@link WorkerLaunched} event to be handled by the corresponding actor.
 * 2. Makes a call to the underlying Mesos driver to launch the task.
 *
 * A task can fail to launch if:
 *
 * 1. It doesn't receive enough metadata for {@link WorkerPorts} to pass its preconditions.
 *      - No launch task request will be made for this assignment result.
 *      - Proactively unschedule the worker.
 * 2. It fails to emit a {@link WorkerLaunched} event.
 *      - The worker will get unscheduled for this launch task request.
 * 3. There are no launch tasks for this assignment result.
 *      - All of these leases are rejected.
 *      - Eventually, the underlying Mesos driver will decline offers since there are no launch task requests.
 *
 * @param requests collection of assignment results received by the scheduler.
 * @param leases list of resource offers from Mesos.
 */
private void launchTasks(Collection<TaskAssignmentResult> requests, List<VirtualMachineLease> leases) {
    List<LaunchTaskRequest> launchTaskRequests = new ArrayList<>();
    for (TaskAssignmentResult assignmentResult : requests) {
        ScheduleRequest request = (ScheduleRequest) assignmentResult.getRequest();
        WorkerPorts workerPorts = null;
        try {
            workerPorts = new WorkerPorts(assignmentResult.getAssignedPorts());
        } catch (IllegalArgumentException | IllegalStateException e) {
            logger.error("problem launching tasks for assignment result {}: {}", assignmentResult, e);
            numMissingWorkerPorts.increment();
        }
        if (workerPorts != null) {
            boolean success = jobMessageRouter.routeWorkerEvent(new WorkerLaunched(request.getWorkerId(), request.getStageNum(), leases.get(0).hostname(), leases.get(0).getVMID(), getAttribute(leases.get(0), slaveClusterAttributeName), workerPorts));
            if (success) {
                launchTaskRequests.add(new LaunchTaskRequest(request, workerPorts));
            } else {
                unscheduleWorker(request.getWorkerId(), Optional.ofNullable(leases.get(0).hostname()));
            }
        } else {
            unscheduleWorker(request.getWorkerId(), Optional.ofNullable(leases.get(0).hostname()));
        }
    }
    if (launchTaskRequests.isEmpty()) {
        for (VirtualMachineLease l : leases) virtualMachineService.rejectLease(l);
    }
    Map<ScheduleRequest, LaunchTaskException> launchErrors = virtualMachineService.launchTasks(launchTaskRequests, leases);
    for (TaskAssignmentResult result : requests) {
        final ScheduleRequest sre = (ScheduleRequest) result.getRequest();
        if (launchErrors.containsKey(sre)) {
            String errorMessage = getWorkerStringPrefix(sre.getStageNum(), sre.getWorkerId()) + " failed due to " + launchErrors.get(sre).getMessage();
            boolean success = jobMessageRouter.routeWorkerEvent(new WorkerLaunchFailed(sre.getWorkerId(), sre.getStageNum(), errorMessage));
            if (!success) {
                logger.warn("Failed to route WorkerLaunchFailed for {} (err {})", sre.getWorkerId(), errorMessage);
            }
        }
    }
}

Example 2 with WorkerLaunched

use of io.mantisrx.server.master.scheduler.WorkerLaunched in project mantis by Netflix.

the class JobClusterManagerTest method testBootStrapJobClustersAndJobs.

@Test
public void testBootStrapJobClustersAndJobs() {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisJobStore jobStore = new MantisJobStore(new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher));
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, false), probe.getRef());
    JobClustersManagerInitializeResponse iResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    List<String> clusterNames = Lists.newArrayList("testBootStrapJobClustersAndJobs1", "testBootStrapJobClustersAndJobs2", "testBootStrapJobClustersAndJobs3");
    String clusterWithNoJob = "testBootStrapJobClusterWithNoJob";
    createJobClusterAndAssert(jobClusterManagerActor, clusterWithNoJob);
    WorkerMigrationConfig migrationConfig = new WorkerMigrationConfig(MigrationStrategyEnum.PERCENTAGE, "{\"percentToMove\":60, \"intervalMs\":30000}");
    // Create 3 clusters and submit 1 job each
    for (String cluster : clusterNames) {
        createJobClusterAndAssert(jobClusterManagerActor, cluster, migrationConfig);
        submitJobAndAssert(jobClusterManagerActor, cluster);
        if (cluster.equals("testBootStrapJobClustersAndJobs1")) {
            // send worker events for job 1 so it goes to started state
            String jobId = "testBootStrapJobClustersAndJobs1-1";
            WorkerId workerId = new WorkerId(jobId, 0, 1);
            WorkerEvent launchedEvent = new WorkerLaunched(workerId, 0, "host1", "vm1", empty(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
            jobClusterManagerActor.tell(launchedEvent, probe.getRef());
            WorkerEvent startInitEvent = new WorkerStatus(new Status(workerId.getJobId(), 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.INFO, "test START_INIT", MantisJobState.StartInitiated));
            jobClusterManagerActor.tell(startInitEvent, probe.getRef());
            WorkerEvent heartBeat = new WorkerHeartbeat(new Status(jobId, 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.HEARTBEAT, "", MantisJobState.Started));
            jobClusterManagerActor.tell(heartBeat, probe.getRef());
            // get Job status
            jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId(jobId).get()), probe.getRef());
            GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
            // Ensure its launched
            assertEquals(SUCCESS, resp2.responseCode);
            assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        }
    }
    // kill 1 of the jobs to test archive path
    JobClusterManagerProto.KillJobRequest killRequest = new JobClusterManagerProto.KillJobRequest("testBootStrapJobClustersAndJobs2-1", JobCompletedReason.Killed.toString(), "njoshi");
    jobClusterManagerActor.tell(killRequest, probe.getRef());
    JobClusterManagerProto.KillJobResponse killJobResponse = probe.expectMsgClass(JobClusterManagerProto.KillJobResponse.class);
    assertEquals(SUCCESS, killJobResponse.responseCode);
    JobTestHelper.sendWorkerTerminatedEvent(probe, jobClusterManagerActor, "testBootStrapJobClustersAndJobs2-1", new WorkerId("testBootStrapJobClustersAndJobs2-1", 0, 1));
    try {
        Thread.sleep(500);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    // Stop job cluster Manager Actor
    system.stop(jobClusterManagerActor);
    // create new instance
    jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    // initialize it
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(JobClustersManagerInitializeResponse.class);
    // probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES),JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    // probe.expectMsgClass(JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    assertEquals(SUCCESS, initializeResponse.responseCode);
    // Get Cluster Config
    jobClusterManagerActor.tell(new GetJobClusterRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetJobClusterResponse clusterResponse = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, clusterResponse.responseCode);
    assertTrue(clusterResponse.getJobCluster().isPresent());
    WorkerMigrationConfig mConfig = clusterResponse.getJobCluster().get().getMigrationConfig();
    assertEquals(migrationConfig.getStrategy(), mConfig.getStrategy());
    assertEquals(migrationConfig.getConfigString(), migrationConfig.getConfigString());
    // get Job status
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs1-1").get()), probe.getRef());
    GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
    // Ensure its launched
    System.out.println("Resp2 -> " + resp2.message);
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
    // 1 jobs should be in completed state
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs2-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobDetailsResponse.class);
    // Ensure its completed
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Completed, resp2.getJobMetadata().get().getState());
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs3-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobDetailsResponse.class);
    // Ensure its Accepted
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
    try {
        Optional<JobWorker> workerByIndex = resp2.getJobMetadata().get().getWorkerByIndex(1, 0);
        assertTrue(workerByIndex.isPresent());
        Optional<IMantisStageMetadata> stageMetadata = resp2.getJobMetadata().get().getStageMetadata(1);
        assertTrue(stageMetadata.isPresent());
        JobWorker workerByIndex1 = stageMetadata.get().getWorkerByIndex(0);
        System.out.println("Got worker by index : " + workerByIndex1);
        Optional<JobWorker> worker = resp2.getJobMetadata().get().getWorkerByNumber(1);
        assertTrue(worker.isPresent());
    } catch (io.mantisrx.server.master.persistence.exceptions.InvalidJobException e) {
        e.printStackTrace();
    }
    jobClusterManagerActor.tell(new GetLastSubmittedJobIdStreamRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetLastSubmittedJobIdStreamResponse lastSubmittedJobIdStreamResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetLastSubmittedJobIdStreamResponse.class);
    lastSubmittedJobIdStreamResponse.getjobIdBehaviorSubject().get().take(1).toBlocking().subscribe((jId) -> {
        assertEquals(new JobId("testBootStrapJobClustersAndJobs1", 1), jId);
    });
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterWithNoJob), probe.getRef());
    GetJobClusterResponse jobClusterResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobClusterResponse.class);
    assertEquals(SUCCESS, jobClusterResponse.responseCode);
    assertTrue(jobClusterResponse.getJobCluster().isPresent());
    assertEquals(clusterWithNoJob, jobClusterResponse.getJobCluster().get().getName());
    // 1 running worker
    verify(schedulerMock, timeout(100_1000).times(1)).initializeRunningWorker(any(), any());
    // 2 worker schedule requests
    verify(schedulerMock, timeout(100_000).times(4)).scheduleWorker(any());
    try {
        Mockito.verify(jobStoreSpied).loadAllArchivedJobsAsync();
        Mockito.verify(jobStoreSpied).loadAllActiveJobs();
        Mockito.verify(jobStoreSpied).loadAllCompletedJobs();
        Mockito.verify(jobStoreSpied).archiveWorker(any());
        Mockito.verify(jobStoreSpied).archiveJob(any());
    } catch (IOException e) {
        e.printStackTrace();
        fail();
    }
}

Example 3 with WorkerLaunched

use of io.mantisrx.server.master.scheduler.WorkerLaunched in project mantis by Netflix.

the class JobTestHelper method sendWorkerLaunchedEvent.

public static void sendWorkerLaunchedEvent(final TestKit probe, final ActorRef jobActor, WorkerId workerId2, int stageNo) {
    WorkerEvent launchedEvent2 = new WorkerLaunched(workerId2, stageNo, "host1", "vm1", Optional.empty(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
    jobActor.tell(launchedEvent2, probe.getRef());
}

Example 4 with WorkerLaunched

use of io.mantisrx.server.master.scheduler.WorkerLaunched in project mantis by Netflix.

the class JobClusterManagerTest method testBootstrapJobClusterAndJobsWithCorruptedWorkerPorts.

/**
 * Case for a master leader re-election when a new master re-hydrates corrupted job worker metadata.
 */
@Test
public void testBootstrapJobClusterAndJobsWithCorruptedWorkerPorts() throws IOException, io.mantisrx.server.master.persistence.exceptions.InvalidJobException {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisJobStore jobStore = new MantisJobStore(new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher));
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, false), probe.getRef());
    probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    String jobClusterName = "testBootStrapJobClustersAndJobs1";
    WorkerMigrationConfig migrationConfig = new WorkerMigrationConfig(MigrationStrategyEnum.PERCENTAGE, "{\"percentToMove\":60, \"intervalMs\":30000}");
    createJobClusterAndAssert(jobClusterManagerActor, jobClusterName, migrationConfig);
    submitJobAndAssert(jobClusterManagerActor, jobClusterName);
    String jobId = "testBootStrapJobClustersAndJobs1-1";
    WorkerId workerId = new WorkerId(jobId, 0, 1);
    WorkerEvent launchedEvent = new WorkerLaunched(workerId, 0, "host1", "vm1", empty(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
    jobClusterManagerActor.tell(launchedEvent, probe.getRef());
    WorkerEvent startInitEvent = new WorkerStatus(new Status(workerId.getJobId(), 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.INFO, "test START_INIT", MantisJobState.StartInitiated));
    jobClusterManagerActor.tell(startInitEvent, probe.getRef());
    WorkerEvent heartBeat = new WorkerHeartbeat(new Status(jobId, 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.HEARTBEAT, "", MantisJobState.Started));
    jobClusterManagerActor.tell(heartBeat, probe.getRef());
    // get Job status
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId(jobId).get()), probe.getRef());
    GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
    // Ensure its launched
    assertEquals(SUCCESS, resp2.responseCode);
    JobWorker worker = new JobWorker.Builder().withWorkerIndex(0).withWorkerNumber(1).withJobId(jobId).withStageNum(1).withNumberOfPorts(5).withWorkerPorts(null).withState(WorkerState.Started).withLifecycleEventsPublisher(eventPublisher).build();
    jobStoreSpied.updateWorker(worker.getMetadata());
    // Stop job cluster Manager Actor
    system.stop(jobClusterManagerActor);
    // create new instance
    jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    // initialize it
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(JobClustersManagerInitializeResponse.class);
    assertEquals(SUCCESS, initializeResponse.responseCode);
    WorkerId newWorkerId = new WorkerId(jobId, 0, 11);
    launchedEvent = new WorkerLaunched(newWorkerId, 0, "host1", "vm1", empty(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
    jobClusterManagerActor.tell(launchedEvent, probe.getRef());
    // Get Cluster Config
    jobClusterManagerActor.tell(new GetJobClusterRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetJobClusterResponse clusterResponse = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, clusterResponse.responseCode);
    assertTrue(clusterResponse.getJobCluster().isPresent());
    WorkerMigrationConfig mConfig = clusterResponse.getJobCluster().get().getMigrationConfig();
    assertEquals(migrationConfig.getStrategy(), mConfig.getStrategy());
    assertEquals(migrationConfig.getConfigString(), migrationConfig.getConfigString());
    // get Job status
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs1-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
    // Ensure its launched
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
    IMantisWorkerMetadata mantisWorkerMetadata = resp2.getJobMetadata().get().getWorkerByIndex(1, 0).get().getMetadata();
    assertNotNull(mantisWorkerMetadata.getWorkerPorts());
    assertEquals(11, mantisWorkerMetadata.getWorkerNumber());
    assertEquals(1, mantisWorkerMetadata.getTotalResubmitCount());
    jobClusterManagerActor.tell(new GetLastSubmittedJobIdStreamRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetLastSubmittedJobIdStreamResponse lastSubmittedJobIdStreamResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetLastSubmittedJobIdStreamResponse.class);
    lastSubmittedJobIdStreamResponse.getjobIdBehaviorSubject().get().take(1).toBlocking().subscribe((jId) -> {
        assertEquals(new JobId("testBootStrapJobClustersAndJobs1", 1), jId);
    });
    // Two schedules: one for the initial success, one for a resubmit from corrupted worker ports.
    verify(schedulerMock, times(2)).scheduleWorker(any());
    // One unschedule from corrupted worker ID 1 (before the resubmit).
    verify(schedulerMock, times(1)).unscheduleAndTerminateWorker(eq(workerId), any());
    try {
        Mockito.verify(jobStoreSpied).loadAllArchivedJobsAsync();
        Mockito.verify(jobStoreSpied).loadAllActiveJobs();
        Mockito.verify(jobStoreSpied).loadAllCompletedJobs();
        Mockito.verify(jobStoreSpied).archiveWorker(any());
    } catch (IOException e) {
        e.printStackTrace();
        fail();
    }
}

Example 5 with WorkerLaunched

use of io.mantisrx.server.master.scheduler.WorkerLaunched in project mantis by Netflix.

the class FakeMantisScheduler method scheduleWorker.

@Override
public void scheduleWorker(final ScheduleRequest scheduleRequest) {
    // Worker Launched
    final WorkerEvent workerLaunched = new WorkerLaunched(scheduleRequest.getWorkerId(), scheduleRequest.getStageNum(), "host1", "vm1", scheduleRequest.getPreferredCluster(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
    jobClusterManagerActor.tell(workerLaunched, ActorRef.noSender());
    // fake Worker Start initiated event
    final WorkerEvent workerStartInit = new WorkerStatus(new Status(scheduleRequest.getWorkerId().getJobId(), scheduleRequest.getStageNum(), scheduleRequest.getWorkerId().getWorkerIndex(), scheduleRequest.getWorkerId().getWorkerNum(), Status.TYPE.INFO, "fake Start Initiated", MantisJobState.StartInitiated));
    jobClusterManagerActor.tell(workerStartInit, ActorRef.noSender());
    // fake Worker Heartbeat event
    final WorkerEvent workerHeartbeat = new WorkerHeartbeat(new Status(scheduleRequest.getWorkerId().getJobId(), scheduleRequest.getStageNum(), scheduleRequest.getWorkerId().getWorkerIndex(), scheduleRequest.getWorkerId().getWorkerNum(), Status.TYPE.HEARTBEAT, "fake heartbeat event", MantisJobState.Started));
    jobClusterManagerActor.tell(workerHeartbeat, ActorRef.noSender());
}