Examples with SchedulingInfo io.mantisrx.runtime.descriptor.SchedulingInfo used on opensource projects

Example 11 with SchedulingInfo

use of io.mantisrx.runtime.descriptor.SchedulingInfo in project mantis by Netflix.

the class VirtualMachineWorkerServiceLocalImpl method createExecuteStageRequest.

private WrappedExecuteStageRequest createExecuteStageRequest() throws MalformedURLException {
    // TODO make ExecuteStageRequest params configurable
    final long timeoutToReportStartSec = 5;
    final URL jobJarUrl = new URL("file:/Users/nmahilani/Projects/Mantis/mantis-sdk/examples/sine-function/build/distributions/sine-function-1.0.zip");
    final List<Integer> ports = Arrays.asList(31015, 31013, 31014);
    final List<Parameter> params = Collections.singletonList(new Parameter("useRandom", "true"));
    final int numInstances = 1;
    // new MachineDefinition(2, 300, 200, 1024, 2), true));
    final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
    final StageSchedulingInfo stage0SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numInstances).machineDefinition(MachineDefinitions.micro()).build();
    final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numInstances).machineDefinition(new MachineDefinition(2, 300, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(1, 1, 5, 1, 1, 30, Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, 15.0, 25.0, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
    // schedulingInfoMap.put(0, stage0SchedInfo);
    schedulingInfoMap.put(1, stage1SchedInfo);
    final SchedulingInfo schedInfo = new SchedulingInfo(schedulingInfoMap);
    final ExecuteStageRequest executeStageRequest = new ExecuteStageRequest(workerInfo.getJobName(), workerInfo.getJobId(), workerInfo.getWorkerIndex(), workerInfo.getWorkerNumber(), jobJarUrl, workerInfo.getStageNumber(), workerInfo.getNumStages(), ports, timeoutToReportStartSec, workerInfo.getMetricsPort(), params, schedInfo, MantisJobDurationType.Transient, 0L, 0L, new WorkerPorts(Arrays.asList(7151, 7152, 7153, 7154, 7155)));
    return new WrappedExecuteStageRequest(PublishSubject.<Boolean>create(), executeStageRequest);
}

Example 12 with SchedulingInfo

use of io.mantisrx.runtime.descriptor.SchedulingInfo in project mantis by Netflix.

the class LocalJobExecutorNetworkedTest method testThreeStageTopology1_2_1.

@Test
public void testThreeStageTopology1_2_1() {
    // 1,2,1 topology
    TestJobThreeStage provider = new TestJobThreeStage();
    SchedulingInfo scheduling = new SchedulingInfo.Builder().numberOfStages(3).singleWorkerStage(MachineDefinitions.micro()).multiWorkerStage(2, MachineDefinitions.micro()).singleWorkerStage(MachineDefinitions.micro()).build();
    LocalJobExecutorNetworked.execute(provider.getJobInstance(), scheduling);
    Iterator<Integer> iter = provider.getItemsWritten().iterator();
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(16, iter.next().intValue());
    Assert.assertEquals(256, iter.next().intValue());
}

Example 13 with SchedulingInfo

use of io.mantisrx.runtime.descriptor.SchedulingInfo in project mantis by Netflix.

the class LocalJobExecutorNetworkedTest method testThreeStageTopology2_1_1.

@Test
public void testThreeStageTopology2_1_1() {
    // 2,1,1 topology
    TestJobThreeStage provider = new TestJobThreeStage();
    SchedulingInfo scheduling = new SchedulingInfo.Builder().numberOfStages(3).multiWorkerStage(2, MachineDefinitions.micro()).singleWorkerStage(MachineDefinitions.micro()).singleWorkerStage(MachineDefinitions.micro()).build();
    LocalJobExecutorNetworked.execute(provider.getJobInstance(), scheduling);
    // with two source instances, should have double the expected
    // input
    Assert.assertEquals(10, provider.getItemsWritten().size());
}

Example 14 with SchedulingInfo

use of io.mantisrx.runtime.descriptor.SchedulingInfo in project mantis by Netflix.

the class SimpleCachedFileStorageProviderTest method testCreateJob.

@Test
public void testCreateJob() {
    String clusterName = "testCreateJob";
    SimpleCachedFileStorageProvider sProvider = new SimpleCachedFileStorageProvider();
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefinition;
    try {
        jobDefinition = JobTestHelper.generateJobDefinition(clusterName);
        JobId jobId = JobId.fromId(clusterName + "-1").get();
        IMantisJobMetadata mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(jobId).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefinition).build();
        sProvider.storeNewJob(mantisJobMetaData);
        SchedulingInfo schedInfo = jobDefinition.getSchedulingInfo();
        int numStages = schedInfo.getStages().size();
        for (int s = 1; s <= numStages; s++) {
            StageSchedulingInfo stage = schedInfo.getStages().get(s);
            IMantisStageMetadata msmd = new MantisStageMetadataImpl.Builder().withJobId(jobId).withStageNum(s).withNumStages(1).withMachineDefinition(stage.getMachineDefinition()).withNumWorkers(stage.getNumberOfInstances()).withHardConstraints(stage.getHardConstraints()).withSoftConstraints(stage.getSoftConstraints()).withScalingPolicy(stage.getScalingPolicy()).isScalable(stage.getScalable()).build();
            ((MantisJobMetadataImpl) mantisJobMetaData).addJobStageIfAbsent(msmd);
            sProvider.updateMantisStage(msmd);
            for (int w = 0; w < stage.getNumberOfInstances(); w++) {
                JobWorker mwmd = new JobWorker.Builder().withJobId(jobId).withWorkerIndex(w).withWorkerNumber(1).withNumberOfPorts(stage.getMachineDefinition().getNumPorts() + MANTIS_SYSTEM_ALLOCATED_NUM_PORTS).withStageNum(w + 1).withLifecycleEventsPublisher(eventPublisher).build();
                ((MantisJobMetadataImpl) mantisJobMetaData).addWorkerMetadata(1, mwmd);
                sProvider.storeWorker(mwmd.getMetadata());
            }
        }
        Optional<IMantisJobMetadata> loadedJobMetaOp = sProvider.loadActiveJob(jobId.getId());
        assertTrue(loadedJobMetaOp.isPresent());
        IMantisJobMetadata loadedJobMeta = loadedJobMetaOp.get();
        System.out.println("Original Job -> " + mantisJobMetaData);
        System.out.println("Loaded Job ->" + loadedJobMeta);
        isEqual(mantisJobMetaData, loadedJobMeta);
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 15 with SchedulingInfo

use of io.mantisrx.runtime.descriptor.SchedulingInfo in project mantis by Netflix.

the class JobAutoScalerTest method testScaleDownNotLessThanMin.

@Test
public void testScaleDownNotLessThanMin() throws InterruptedException {
    final String jobId = "test-job-1";
    final int coolDownSec = 2;
    final int scalingStageNum = 1;
    final MantisMasterClientApi mockMasterClientApi = mock(MantisMasterClientApi.class);
    final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
    final int numStage1Workers = 5;
    final int increment = 10;
    // decrement by 10 on scale down, this will push num workers below min and below 0.
    final int decrement = 10;
    final int min = 3;
    final int max = 50;
    final double scaleUpAbovePct = 45.0;
    final double scaleDownBelowPct = 15.0;
    final double workerMemoryMB = 512.0;
    final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numStage1Workers).machineDefinition(new MachineDefinition(2, workerMemoryMB, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(scalingStageNum, min, max, increment, decrement, coolDownSec, Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, scaleDownBelowPct, scaleUpAbovePct, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
    schedulingInfoMap.put(scalingStageNum, stage1SchedInfo);
    when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), anyInt(), anyString())).thenReturn(Observable.just(true));
    Context context = mock(Context.class);
    when(context.getWorkerMapObservable()).thenReturn(Observable.empty());
    final JobAutoScaler jobAutoScaler = new JobAutoScaler(jobId, new SchedulingInfo(schedulingInfoMap), mockMasterClientApi, context);
    jobAutoScaler.start();
    final Observer<JobAutoScaler.Event> jobAutoScalerObserver = jobAutoScaler.getObserver();
    // should trigger a scale down (below 15% scaleDown threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleDownBelowPct / 100.0 - 0.01), numStage1Workers, ""));
    verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, min, String.format("Memory with value %1$,.2f is below scaleDown threshold of %2$,.1f", (scaleDownBelowPct / 100.0 - 0.01) * 100.0, scaleDownBelowPct));
    verifyNoMoreInteractions(mockMasterClientApi);
}