Examples with JobSla io.mantisrx.runtime.JobSla used on opensource projects

Example 1 with JobSla

use of io.mantisrx.runtime.JobSla in project mantis by Netflix.

the class TestHelpers method createFakeScheduleRequest.

public static ScheduleRequest createFakeScheduleRequest(final WorkerId workerId, final int stageNum, final int numStages, final MachineDefinition machineDefinition) {
    try {
        JobDefinition jobDefinition = new JobDefinition.Builder().withArtifactName("jar").withSchedulingInfo(new SchedulingInfo(Collections.singletonMap(0, StageSchedulingInfo.builder().numberOfInstances(1).machineDefinition(machineDefinition).hardConstraints(Collections.emptyList()).softConstraints(Collections.emptyList()).build()))).withJobSla(new JobSla(0, 0, null, MantisJobDurationType.Perpetual, null)).build();
        IMantisJobMetadata mantisJobMetadata = new MantisJobMetadataImpl.Builder().withJobId(JobId.fromId(workerId.getJobId()).get()).withJobDefinition(jobDefinition).build();
        return new ScheduleRequest(workerId, stageNum, numStages, new JobMetadata(mantisJobMetadata.getJobId().getId(), mantisJobMetadata.getJobJarUrl(), mantisJobMetadata.getTotalStages(), mantisJobMetadata.getUser(), mantisJobMetadata.getSchedulingInfo(), mantisJobMetadata.getParameters(), mantisJobMetadata.getSubscriptionTimeoutSecs(), mantisJobMetadata.getMinRuntimeSecs()), mantisJobMetadata.getSla().get().getDurationType(), machineDefinition, Collections.emptyList(), Collections.emptyList(), 0, Optional.empty());
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}

Example 2 with JobSla

use of io.mantisrx.runtime.JobSla in project mantis by Netflix.

the class SubmitWithRuntimeLimit method main.

public static void main(String[] args) {
    try {
        Args.parse(SubmitWithRuntimeLimit.class, args);
    } catch (IllegalArgumentException e) {
        Args.usage(SubmitEphemeralJob.class);
        System.exit(1);
    }
    Properties properties = new Properties();
    try (InputStream inputStream = new FileInputStream(propFile)) {
        properties.load(inputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicBoolean completed = new AtomicBoolean(false);
    final long runtimeLimitSecs = 30;
    MantisSSEJob job = new MantisSSEJob.Builder(properties).name(jobName).jobSla(new JobSla(runtimeLimitSecs, 0L, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Perpetual, "")).onConnectionReset(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            System.err.println("Reconnecting due to error: " + throwable.getMessage());
        }
    }).buildJobSubmitter();
    final Observable<Observable<MantisServerSentEvent>> observable = job.submitAndGet();
    final Subscription subscription = observable.flatMap(new Func1<Observable<MantisServerSentEvent>, Observable<?>>() {

        @Override
        public Observable<?> call(Observable<MantisServerSentEvent> eventObservable) {
            return eventObservable.doOnNext(new Action1<MantisServerSentEvent>() {

                @Override
                public void call(MantisServerSentEvent event) {
                    if (completed.get())
                        System.out.println("FAILURE");
                    System.out.println("Got: " + event.getEventAsString());
                }
            });
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            latch.countDown();
        }
    }).subscribe();
    try {
        // add a buffer for job launch time
        Thread.sleep((runtimeLimitSecs + 10) * 1000);
        // set expectation of complete
        completed.set(true);
        // give some time to see if we still get data, which would be a failure
        Thread.sleep(20000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    subscription.unsubscribe();
    System.exit(0);
}

Example 3 with JobSla

use of io.mantisrx.runtime.JobSla in project mantis by Netflix.

the class JobClusterTest method testQuickJobSubmitWithNoSchedInfoInPreviousJob.

@Test
public void testQuickJobSubmitWithNoSchedInfoInPreviousJob() {
    TestKit probe = new TestKit(system);
    String clusterName = "testQuickJobSubmitWithNoSchedInfoInPreviousJob";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    try {
        // job defn with scheduling info
        final JobDefinition jobDefn = new JobDefinition.Builder().withName(clusterName).withParameters(Lists.newArrayList()).withLabels(Lists.newArrayList()).withVersion("0.0.1").withSubscriptionTimeoutSecs(300).withUser("njoshi").withJobSla(new JobSla(0, 0, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Transient, "abc")).build();
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, jobId, SUCCESS, JobState.Accepted);
        // submit another job this time with no job definition
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, null, clusterName + "-2");
        JobTestHelper.getJobDetailsAndVerify(probe, jobClusterActor, clusterName + "-2", SUCCESS, JobState.Accepted);
        JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 1), jobClusterActor);
        verify(jobStoreMock, times(1)).createJobCluster(any());
        verify(jobStoreMock, times(2)).updateJobCluster(any());
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    }
// Mockito.doThrow(IOException.class).when(jobStoreMock).storeNewJob(any());
}

Example 4 with JobSla

use of io.mantisrx.runtime.JobSla in project mantis by Netflix.

the class JobClusterTest method testJobSubmitWithVersionAndNoSchedInfo.

@Test
public void testJobSubmitWithVersionAndNoSchedInfo() {
    TestKit probe = new TestKit(system);
    String clusterName = "testJobSubmitWithVersionAndNoSchedInfo";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    final JobClusterDefinitionImpl fakeJobCluster = createFakeJobClusterDefn(clusterName);
    ActorRef jobClusterActor = system.actorOf(props(clusterName, jobStoreMock, schedulerMock, eventPublisher));
    jobClusterActor.tell(new JobClusterProto.InitializeJobClusterRequest(fakeJobCluster, user, probe.getRef()), probe.getRef());
    JobClusterProto.InitializeJobClusterResponse createResp = probe.expectMsgClass(JobClusterProto.InitializeJobClusterResponse.class);
    assertEquals(SUCCESS, createResp.responseCode);
    JobClusterConfig clusterConfig = new JobClusterConfig.Builder().withArtifactName("myart2").withSchedulingInfo(TWO_WORKER_SCHED_INFO).withVersion("0.0.2").build();
    final JobClusterDefinitionImpl updatedFakeJobCluster = new JobClusterDefinitionImpl.Builder().withJobClusterConfig(clusterConfig).withName(clusterName).withParameters(Lists.newArrayList()).withUser(user).withIsReadyForJobMaster(true).withOwner(DEFAULT_JOB_OWNER).withMigrationConfig(WorkerMigrationConfig.DEFAULT).withSla(NO_OP_SLA).build();
    jobClusterActor.tell(new UpdateJobClusterRequest(updatedFakeJobCluster, "user"), probe.getRef());
    UpdateJobClusterResponse resp = probe.expectMsgClass(UpdateJobClusterResponse.class);
    jobClusterActor.tell(new GetJobClusterRequest(clusterName), probe.getRef());
    GetJobClusterResponse getJobClusterResponse = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(2, getJobClusterResponse.getJobCluster().get().getJars().size());
    try {
        final JobDefinition jobDefn = new JobDefinition.Builder().withName(clusterName).withParameters(Lists.newArrayList()).withUser("njoshi").withVersion("0.0.2").withSubscriptionTimeoutSecs(300).withJobSla(new JobSla(0, 0, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Transient, "")).build();
        ;
        String jobId = clusterName + "-1";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn, jobId);
        jobClusterActor.tell(new GetJobDetailsRequest("nj", JobId.fromId(jobId).get()), probe.getRef());
        GetJobDetailsResponse detailsResp = probe.expectMsgClass(GetJobDetailsResponse.class);
        // make sure it inherits from cluster
        assertEquals("myart2", detailsResp.getJobMetadata().get().getArtifactName());
        // inherits cluster scheduling Info corresponding to the given artifact
        assertEquals(TWO_WORKER_SCHED_INFO, detailsResp.getJobMetadata().get().getSchedulingInfo());
        // Now submit with a different artifact and no scheduling Info
        final JobDefinition jobDefn2 = new JobDefinition.Builder().withName(clusterName).withParameters(Lists.newArrayList()).withUser("njoshi").withVersion("0.0.1").withSubscriptionTimeoutSecs(300).withJobSla(new JobSla(0, 0, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Transient, "")).build();
        ;
        String jobId2 = clusterName + "-2";
        JobTestHelper.submitJobAndVerifySuccess(probe, clusterName, jobClusterActor, jobDefn2, jobId2);
        jobClusterActor.tell(new GetJobDetailsRequest("nj", JobId.fromId(jobId2).get()), probe.getRef());
        GetJobDetailsResponse detailsResp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
        // make sure it inherits from cluster
        assertEquals("myart", detailsResp2.getJobMetadata().get().getArtifactName());
        // inherits cluster scheduling Info corresponding to the given artifact
        assertEquals(SINGLE_WORKER_SCHED_INFO, detailsResp2.getJobMetadata().get().getSchedulingInfo());
        JobTestHelper.killJobAndVerify(probe, clusterName, new JobId(clusterName, 2), jobClusterActor);
        verify(jobStoreMock, times(1)).createJobCluster(any());
        verify(jobStoreMock, times(3)).updateJobCluster(any());
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        fail();
    }
// Mockito.doThrow(IOException.class).when(jobStoreMock).storeNewJob(any());
}

Example 5 with JobSla

use of io.mantisrx.runtime.JobSla in project mantis by Netflix.

the class DataFormatAdapterTest method convertMantisJobWriteableTest.

@Test
public void convertMantisJobWriteableTest() throws Exception {
    String artifactName = "artifact";
    String version = "1.0.0";
    String clusterName = "myCluster";
    List<Label> labels = new ArrayList<>();
    Label label = new Label("myLable", "myVal");
    labels.add(label);
    List<Parameter> params = new ArrayList<>();
    Parameter param = new Parameter("myparam", "myval");
    params.add(param);
    long subTimeout = 1000;
    JobSla jobSla = new JobSla(100, 10, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Perpetual, "userType");
    JobDefinition jobDefn = new JobDefinition.Builder().withArtifactName(artifactName).withName(clusterName).withLabels(labels).withParameters(params).withSchedulingInfo(DEFAULT_SCHED_INFO).withUser("user").withJobSla(jobSla).withSubscriptionTimeoutSecs(subTimeout).withNumberOfStages(DEFAULT_SCHED_INFO.getStages().size()).build();
    JobId jobId = new JobId(clusterName, 1);
    long currTime = System.currentTimeMillis();
    Instant startedAt = Instant.ofEpochMilli(currTime);
    Instant endedAt = startedAt.plusSeconds(5);
    Instant submittedAt = startedAt.minusSeconds(5);
    IMantisJobMetadata jobmeta = new MantisJobMetadataImpl.Builder().withJobDefinition(jobDefn).withJobId(jobId).withNextWorkerNumToUse(2).withSubmittedAt(submittedAt).withJobState(JobState.Launched).build();
    IMantisWorkerMetadata workerMetadata = new MantisWorkerMetadataImpl(0, 1, jobId.getId(), 1, 3, new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)), WorkerState.Started, "slave", "slaveId", startedAt.toEpochMilli(), startedAt.toEpochMilli(), startedAt.toEpochMilli(), startedAt.toEpochMilli(), -1, JobCompletedReason.Normal, 0, 0, of("cluster"));
    ((MantisJobMetadataImpl) jobmeta).addJobStageIfAbsent(new MantisStageMetadataImpl.Builder().withNumStages(1).withStageNum(1).withNumWorkers(1).withJobId(jobId).withHardConstraints(Lists.newArrayList()).withSoftConstraints(Lists.newArrayList()).withMachineDefinition(DEFAULT_MACHINE_DEFINITION).build());
    ((MantisJobMetadataImpl) jobmeta).addWorkerMetadata(1, new JobWorker(workerMetadata, eventPublisher));
    MantisJobMetadata oldFormat = DataFormatAdapter.convertMantisJobMetadataToMantisJobMetadataWriteable(jobmeta);
    System.out.println("oldForamt -> " + oldFormat);
    assertEquals(jobId.getId(), oldFormat.getJobId());
    assertEquals(label, oldFormat.getLabels().get(0));
    assertEquals(param, oldFormat.getParameters().get(0));
    assertEquals(clusterName, oldFormat.getName());
    assertEquals(jobSla, oldFormat.getSla());
    assertEquals(1, oldFormat.getNumStages());
    assertEquals(subTimeout, oldFormat.getSubscriptionTimeoutSecs());
    assertEquals(2, oldFormat.getNextWorkerNumberToUse());
    assertEquals("http://" + artifactName, oldFormat.getJarUrl().toString());
    assertEquals(MantisJobState.Launched, oldFormat.getState());
    assertEquals(submittedAt.toEpochMilli(), oldFormat.getSubmittedAt());
    assertEquals("user", oldFormat.getUser());
    IMantisJobMetadata reconverted = DataFormatAdapter.convertMantisJobWriteableToMantisJobMetadata(oldFormat, eventPublisher);
    System.out.println("newForamt -> " + reconverted);
    // assertEquals(jobmeta, reconverted);
    // assertTrue(jobmeta.equals(reconverted));
    assertEquals(jobmeta.getArtifactName(), reconverted.getArtifactName());
    assertEquals(jobmeta.getClusterName(), reconverted.getClusterName());
    System.out.println("expected Jobdef " + jobmeta.getJobDefinition());
    System.out.println("actual   Jobdef " + reconverted.getJobDefinition());
    assertEquals(jobmeta.getJobDefinition(), reconverted.getJobDefinition());
    assertEquals(jobmeta.getJobId(), reconverted.getJobId());
    assertEquals(jobmeta.getJobJarUrl(), reconverted.getJobJarUrl());
    assertEquals(jobmeta.getLabels().get(0), reconverted.getLabels().get(0));
    assertEquals(jobmeta.getParameters().get(0), reconverted.getParameters().get(0));
    assertEquals(jobmeta.getMinRuntimeSecs(), reconverted.getMinRuntimeSecs());
    assertEquals(jobmeta.getNextWorkerNumberToUse(), reconverted.getNextWorkerNumberToUse());
    assertEquals(jobmeta.getSla().get(), reconverted.getSla().get());
    assertEquals(jobmeta.getSubmittedAtInstant(), reconverted.getSubmittedAtInstant());
    assertEquals(jobmeta.getState(), reconverted.getState());
    assertEquals(jobmeta.getSubscriptionTimeoutSecs(), reconverted.getSubscriptionTimeoutSecs());
    assertEquals(jobmeta.getTotalStages(), reconverted.getTotalStages());
    assertEquals(jobmeta.getUser(), reconverted.getUser());
// assertEquals(jobmeta.getSchedulingInfo(), reconverted.getSchedulingInfo());
}