Examples with JobMetadata io.mantisrx.server.core.domain.JobMetadata used on opensource projects

Example 1 with JobMetadata

use of io.mantisrx.server.core.domain.JobMetadata in project mantis by Netflix.

the class TestHelpers method createFakeScheduleRequest.

public static ScheduleRequest createFakeScheduleRequest(final WorkerId workerId, final int stageNum, final int numStages, final MachineDefinition machineDefinition) {
    try {
        JobDefinition jobDefinition = new JobDefinition.Builder().withArtifactName("jar").withSchedulingInfo(new SchedulingInfo(Collections.singletonMap(0, StageSchedulingInfo.builder().numberOfInstances(1).machineDefinition(machineDefinition).hardConstraints(Collections.emptyList()).softConstraints(Collections.emptyList()).build()))).withJobSla(new JobSla(0, 0, null, MantisJobDurationType.Perpetual, null)).build();
        IMantisJobMetadata mantisJobMetadata = new MantisJobMetadataImpl.Builder().withJobId(JobId.fromId(workerId.getJobId()).get()).withJobDefinition(jobDefinition).build();
        return new ScheduleRequest(workerId, stageNum, numStages, new JobMetadata(mantisJobMetadata.getJobId().getId(), mantisJobMetadata.getJobJarUrl(), mantisJobMetadata.getTotalStages(), mantisJobMetadata.getUser(), mantisJobMetadata.getSchedulingInfo(), mantisJobMetadata.getParameters(), mantisJobMetadata.getSubscriptionTimeoutSecs(), mantisJobMetadata.getMinRuntimeSecs()), mantisJobMetadata.getSla().get().getDurationType(), machineDefinition, Collections.emptyList(), Collections.emptyList(), 0, Optional.empty());
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}

Example 2 with JobMetadata

use of io.mantisrx.server.core.domain.JobMetadata in project mantis by Netflix.

the class VirtualMachineMasterServiceMesosImpl method createMantisWorkerExecutor.

private ExecutorInfo createMantisWorkerExecutor(final ExecuteStageRequest executeStageRequest, final LaunchTaskRequest launchTaskRequest, final double memoryMB, final double cpuCores) {
    final int memSize = getMemSize((int) memoryMB);
    final int numCpu = (int) Math.ceil(cpuCores);
    final WorkerId workerId = launchTaskRequest.getScheduleRequest().getWorkerId();
    String executorName = workerId.getId();
    JobMetadata jobMetadata = launchTaskRequest.getScheduleRequest().getJobMetadata();
    URL jobJarUrl = jobMetadata.getJobJarUrl();
    Protos.Environment.Builder envBuilder = Protos.Environment.newBuilder().addVariables(Protos.Environment.Variable.newBuilder().setName("JOB_URL").setValue(jobJarUrl.toString())).addVariables(Protos.Environment.Variable.newBuilder().setName("JOB_NAME").setValue(executorName)).addVariables(Protos.Environment.Variable.newBuilder().setName("WORKER_LIB_DIR").setValue(getWorkerLibDir())).addVariables(Protos.Environment.Variable.newBuilder().setName("JVM_MEMORY_MB").setValue("" + (memSize))).addVariables(Protos.Environment.Variable.newBuilder().setName("JVM_META_SPACE_MB").setValue("100")).addVariables(Protos.Environment.Variable.newBuilder().setName("JVM_CODE_CACHE_SIZE_MB").setValue("200")).addVariables(Protos.Environment.Variable.newBuilder().setName("JVM_COMP_CLASS_SIZE_MB").setValue("100")).addVariables(Protos.Environment.Variable.newBuilder().setName("WORKER_INDEX").setValue("" + (workerId.getWorkerIndex()))).addVariables(Protos.Environment.Variable.newBuilder().setName("WORKER_NUMBER").setValue("" + (workerId.getWorkerNum()))).addVariables(Protos.Environment.Variable.newBuilder().setName("JOB_ID").setValue(workerId.getJobId())).addVariables(Protos.Environment.Variable.newBuilder().setName("MANTIS_WORKER_DEBUG_PORT").setValue("" + launchTaskRequest.getPorts().getDebugPort())).addVariables(Protos.Environment.Variable.newBuilder().setName("MANTIS_WORKER_CONSOLE_PORT").setValue("" + launchTaskRequest.getPorts().getConsolePort())).addVariables(Protos.Environment.Variable.newBuilder().setName("MANTIS_USER").setValue("" + jobMetadata.getUser())).addVariables(Protos.Environment.Variable.newBuilder().setName("STAGE_NUMBER").setValue("" + launchTaskRequest.getScheduleRequest().getStageNum())).addVariables(Protos.Environment.Variable.newBuilder().setName("NUM_CPU").setValue("" + numCpu));
    // add worker info
    Map<String, String> envVars = new WorkerTopologyInfo.Writer(executeStageRequest).getEnvVars();
    for (Map.Entry<String, String> entry : envVars.entrySet()) {
        envBuilder = envBuilder.addVariables(Protos.Environment.Variable.newBuilder().setName(entry.getKey()).setValue(entry.getValue()));
    }
    // add job parameters
    for (Parameter parameter : executeStageRequest.getParameters()) {
        if (parameter.getName() != null && parameter.getValue() != null) {
            envBuilder = envBuilder.addVariables(Protos.Environment.Variable.newBuilder().setName(String.format("JOB_PARAM_" + parameter.getName())).setValue(parameter.getValue()));
        }
    }
    // add ZooKeeper properties
    Protos.Environment env = envBuilder.addVariables(Protos.Environment.Variable.newBuilder().setName("mantis.zookeeper.connectString").setValue(masterConfig.getZkConnectionString())).addVariables(Protos.Environment.Variable.newBuilder().setName("mantis.zookeeper.root").setValue(masterConfig.getZkRoot())).addVariables(Protos.Environment.Variable.newBuilder().setName("mantis.zookeeper.leader.announcement.path").setValue(masterConfig.getLeaderAnnouncementPath())).addVariables(Protos.Environment.Variable.newBuilder().setName("MASTER_DESCRIPTION").setValue(masterDescriptionJson)).build();
    return ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue(executorName)).setCommand(CommandInfo.newBuilder().setValue(getWorkerExecutorStartupScriptFullPath()).setEnvironment(env)).setName(getWorkerExecutorName()).setSource(workerId.getJobId()).build();
}

Example 3 with JobMetadata

use of io.mantisrx.server.core.domain.JobMetadata in project mantis by Netflix.

the class JobTestLifecycle method testJobSubmitPerpetual.

@Test
public void testJobSubmitPerpetual() {
    final TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitPerpetual";
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefn;
    try {
        MachineDefinition machineDefinition = new MachineDefinition(1.0, 1.0, 1.0, 1.0, 3);
        SchedulingInfo schedInfo = new SchedulingInfo.Builder().numberOfStages(1).singleWorkerStageWithConstraints(machineDefinition, Lists.newArrayList(), Lists.newArrayList()).build();
        jobDefn = new JobDefinition.Builder().withName(clusterName).withParameters(Lists.newArrayList()).withLabels(Lists.newArrayList()).withSchedulingInfo(schedInfo).withArtifactName("myart").withSubscriptionTimeoutSecs(30).withUser("njoshi").withNumberOfStages(schedInfo.getStages().size()).withJobSla(new JobSla(0, 0, null, MantisJobDurationType.Perpetual, null)).build();
        MantisScheduler schedulerMock = mock(MantisScheduler.class);
        MantisJobStore jobStoreMock = mock(MantisJobStore.class);
        MantisJobMetadataImpl mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(new JobId(clusterName, 1)).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefn).build();
        final ActorRef jobActor = system.actorOf(JobActor.props(jobClusterDefn, mantisJobMetaData, jobStoreMock, schedulerMock, eventPublisher));
        jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        JobProto.JobInitialized initMsg = probe.expectMsgClass(JobProto.JobInitialized.class);
        assertEquals(SUCCESS, initMsg.responseCode);
        String jobId = clusterName + "-1";
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp + " msg " + resp.message);
        assertEquals(SUCCESS, resp.responseCode);
        assertEquals(JobState.Accepted, resp.getJobMetadata().get().getState());
        assertTrue(resp.getJobMetadata().get().getStageMetadata(1).isPresent());
        // send launched event
        WorkerId workerId = new WorkerId(jobId, 0, 1);
        int stageNum = 1;
        JobTestHelper.sendWorkerLaunchedEvent(probe, jobActor, workerId, stageNum);
        JobTestHelper.sendStartInitiatedEvent(probe, jobActor, stageNum, workerId);
        // send heartbeat
        JobTestHelper.sendHeartBeat(probe, jobActor, jobId, stageNum, workerId);
        // check job status again
        jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("nj", jobId), probe.getRef());
        // jobActor.tell(new JobProto.InitJob(probe.getRef()), probe.getRef());
        GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        System.out.println("resp " + resp2 + " msg " + resp2.message);
        assertEquals(SUCCESS, resp2.responseCode);
        assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        verify(jobStoreMock, times(1)).storeNewJob(any());
        verify(jobStoreMock, times(1)).storeNewWorkers(any(), any());
        verify(jobStoreMock, times(3)).updateWorker(any());
        verify(jobStoreMock, times(3)).updateJob(any());
        // verify(jobStoreMock, times(3))
        verify(schedulerMock, times(1)).scheduleWorker(any());
        JobMetadata jobMetadata = new JobMetadata(jobId, new URL("http://myart" + ""), 1, "njoshi", schedInfo, Lists.newArrayList(), 0, 0);
        ScheduleRequest expectedScheduleRequest = new ScheduleRequest(workerId, 1, 4, jobMetadata, MantisJobDurationType.Perpetual, machineDefinition, Lists.newArrayList(), Lists.newArrayList(), 0, empty());
        verify(schedulerMock).scheduleWorker(expectedScheduleRequest);
    // assertEquals(jobActor, probe.getLastSender());
    } catch (InvalidJobException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}