Examples with Context io.mantisrx.runtime.Context used on opensource projects

Example 21 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class JobAutoScalerTest method testScaleUpOnDifferentScalingReasons.

@Test
public void testScaleUpOnDifferentScalingReasons() throws InterruptedException {
    final List<StageScalingPolicy.ScalingReason> scalingReasons = Arrays.asList(DataDrop, KafkaLag, UserDefined);
    for (StageScalingPolicy.ScalingReason scalingReason : scalingReasons) {
        logger.info("==== test scaling reason {} =====", scalingReason.name());
        final String jobId = "test-job-1";
        final int coolDownSec = 2;
        final int scalingStageNum = 1;
        final MantisMasterClientApi mockMasterClientApi = mock(MantisMasterClientApi.class);
        final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
        final int numStage1Workers = 1;
        final int increment = 1;
        final int decrement = 0;
        final int min = 1;
        final int max = 5;
        final double scaleUpAbove = 2000.0;
        final double scaleDownBelow = 0.0;
        final double workerMemoryMB = 512.0;
        final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numStage1Workers).machineDefinition(new MachineDefinition(2, workerMemoryMB, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(scalingStageNum, min, max, increment, decrement, coolDownSec, Collections.singletonMap(scalingReason, new StageScalingPolicy.Strategy(scalingReason, scaleDownBelow, scaleUpAbove, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
        schedulingInfoMap.put(scalingStageNum, stage1SchedInfo);
        when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), eq(numStage1Workers + increment), anyString())).thenReturn(Observable.just(true));
        Context context = mock(Context.class);
        when(context.getWorkerMapObservable()).thenReturn(Observable.empty());
        final JobAutoScaler jobAutoScaler = new JobAutoScaler(jobId, new SchedulingInfo(schedulingInfoMap), mockMasterClientApi, context);
        jobAutoScaler.start();
        final Observer<JobAutoScaler.Event> jobAutoScalerObserver = jobAutoScaler.getObserver();
        // should trigger a scale up (above scaleUp threshold)
        jobAutoScalerObserver.onNext(new JobAutoScaler.Event(scalingReason, scalingStageNum, scaleUpAbove + 0.01, numStage1Workers, ""));
        verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, numStage1Workers + increment, String.format("%s with value %2$.2f exceeded scaleUp threshold of %3$.1f", scalingReason.name(), (scaleUpAbove + 0.01), scaleUpAbove));
        // should *not* trigger a scale up before cooldown period (above scaleUp threshold)
        jobAutoScalerObserver.onNext(new JobAutoScaler.Event(scalingReason, scalingStageNum, scaleUpAbove + 0.01, numStage1Workers + increment, ""));
        jobAutoScalerObserver.onNext(new JobAutoScaler.Event(scalingReason, scalingStageNum, scaleUpAbove + 0.01, numStage1Workers + increment, ""));
        Thread.sleep(coolDownSec * 1000);
        // retry sending auto scale event till scaleJobStage request sent to master, as there is possible a race between the sleep for coolDownSecs in the Test and the event being processed before coolDownSecs
        final CountDownLatch retryLatch = new CountDownLatch(1);
        when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), eq(numStage1Workers + 2 * increment), anyString())).thenAnswer(new Answer<Observable<Void>>() {

            @Override
            public Observable<Void> answer(InvocationOnMock invocation) throws Throwable {
                retryLatch.countDown();
                return Observable.just(null);
            }
        });
        do {
            logger.info("sending Job auto scale Event");
            // should trigger a scale up after cooldown period (above scaleUp threshold)
            jobAutoScalerObserver.onNext(new JobAutoScaler.Event(scalingReason, scalingStageNum, scaleUpAbove + 0.01, numStage1Workers + increment, ""));
        } while (!retryLatch.await(1, TimeUnit.SECONDS));
        verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, numStage1Workers + 2 * increment, String.format("%s with value %2$.2f exceeded scaleUp threshold of %3$.1f", scalingReason.name(), (scaleUpAbove + 0.01), scaleUpAbove));
    }
}

Example 22 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class JobAutoScalerTest method testScalingResiliency.

@Test
public void testScalingResiliency() throws InterruptedException {
    final String jobId = "test-job-1";
    final int coolDownSec = 2;
    final int scalingStageNum = 1;
    final MantisMasterClientApi mockMasterClientApi = mock(MantisMasterClientApi.class);
    final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
    final int numStage1Workers = 1;
    final int increment = 1;
    final int decrement = 1;
    final int min = 1;
    final int max = 5;
    final double scaleUpAbovePct = 45.0;
    final double scaleDownBelowPct = 15.0;
    final double workerMemoryMB = 512.0;
    final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numStage1Workers).machineDefinition(new MachineDefinition(2, workerMemoryMB, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(scalingStageNum, min, max, increment, decrement, coolDownSec, Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, scaleDownBelowPct, scaleUpAbovePct, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
    schedulingInfoMap.put(scalingStageNum, stage1SchedInfo);
    final CountDownLatch scaleJobStageSuccessLatch = new CountDownLatch(1);
    final AtomicInteger count = new AtomicInteger(0);
    final Observable<Boolean> simulateScaleJobStageFailureResp = Observable.just(1).map(new Func1<Integer, Boolean>() {

        @Override
        public Boolean call(Integer integer) {
            if (count.incrementAndGet() < 3) {
                throw new IllegalStateException("fake connection exception");
            } else {
                scaleJobStageSuccessLatch.countDown();
                return true;
            }
        }
    });
    when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), eq(numStage1Workers + increment), anyString())).thenReturn(simulateScaleJobStageFailureResp);
    Context context = mock(Context.class);
    when(context.getWorkerMapObservable()).thenReturn(Observable.empty());
    final JobAutoScaler jobAutoScaler = new JobAutoScaler(jobId, new SchedulingInfo(schedulingInfoMap), mockMasterClientApi, context);
    jobAutoScaler.start();
    final Observer<JobAutoScaler.Event> jobAutoScalerObserver = jobAutoScaler.getObserver();
    // should trigger a scale up (above 45% scaleUp threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleUpAbovePct / 100.0 + 0.01), numStage1Workers, ""));
    verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, numStage1Workers + increment, String.format("Memory with value %1$,.2f exceeded scaleUp threshold of 45.0", (scaleUpAbovePct / 100.0 + 0.01) * 100.0));
    scaleJobStageSuccessLatch.await();
}

Example 23 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class StageExecutorsSingleStageTest method testSingleStageJob.

@SuppressWarnings("rawtypes")
@Test
public void testSingleStageJob() {
    Action0 noOpAction = new Action0() {

        @Override
        public void call() {
        }
    };
    Action1<Throwable> noOpError = new Action1<Throwable>() {

        @Override
        public void call(Throwable t) {
        }
    };
    TestJobSingleStage provider = new TestJobSingleStage();
    Job job = provider.getJobInstance();
    PortSelector portSelector = new PortSelectorInRange(8000, 9000);
    StageConfig<?, ?> stage = (StageConfig<?, ?>) job.getStages().get(0);
    BehaviorSubject<Integer> workersInStageOneObservable = BehaviorSubject.create(1);
    StageExecutors.executeSingleStageJob(job.getSource(), stage, job.getSink(), portSelector, new RxMetrics(), new Context(), noOpAction, 0, workersInStageOneObservable, null, null, noOpAction, noOpError);
    Iterator<Integer> iter = provider.getItemsWritten().iterator();
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(1, iter.next().intValue());
    Assert.assertEquals(4, iter.next().intValue());
    Assert.assertEquals(9, iter.next().intValue());
}

Example 24 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class TestJobParameterized method getJobInstance.

@Override
public Job<Integer> getJobInstance() {
    return MantisJob.<Integer>source(new Source<Integer>() {

        @Override
        public Observable<Observable<Integer>> call(Context context, Index index) {
            Integer start = (Integer) context.getParameters().get("start-range");
            Integer end = (Integer) context.getParameters().get("end-range");
            return Observable.just(Observable.range(start, end));
        }
    }).stage(new ScalarComputation<Integer, Integer>() {

        @Override
        public Observable<Integer> call(Context context, Observable<Integer> t1) {
            final Integer scale = (Integer) context.getParameters().get("scale-by");
            return t1.map(new Func1<Integer, Integer>() {

                @Override
                public Integer call(Integer t1) {
                    return t1 * scale;
                }
            });
        }
    }, new ScalarToScalar.Config<Integer, Integer>().codec(Codecs.integer())).sink(new Sink<Integer>() {

        @Override
        public void init(Context context) {
            System.out.println("sink init called");
        }

        @Override
        public void call(Context context, PortRequest p, Observable<Integer> o) {
            final String message = (String) context.getParameters().get("sink-message");
            o.toBlocking().forEach(new Action1<Integer>() {

                @Override
                public void call(Integer t1) {
                    System.out.println(message + t1);
                    itemsWritten.add(t1);
                }
            });
        }
    }).metadata(new Metadata.Builder().name("test job").description("showcase parameters").build()).parameterDefinition(new IntParameter().name("start-range").validator(Validators.range(0, 100)).build()).parameterDefinition(new IntParameter().name("end-range").validator(Validators.range(100, 1000)).build()).parameterDefinition(new IntParameter().name("scale-by").description("scale each value from the range").validator(Validators.range(1, 10)).build()).parameterDefinition(new StringParameter().name("sink-message").defaultValue("hello: ").description("concat with results").validator(Validators.notNullOrEmpty()).build()).create();
}

Example 25 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class ContextualHttpSourceTest method canStreamFromMultipleServersWithCorrectContext.

@Test
public void canStreamFromMultipleServersWithCorrectContext() throws Exception {
    ContextualHttpSource<ServerSentEvent> source = HttpSources.contextualSource(HttpClientFactories.sseClientFactory(), HttpRequestFactories.createGetFactory("test/stream")).withServerProvider(localServerProvider).withActivityObserver(sourceObserver).build();
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    final CopyOnWriteArraySet<ServerInfo> connectedServers = new CopyOnWriteArraySet<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerContext<ServerSentEvent>>() {

        @Override
        public void call(ServerContext<ServerSentEvent> pair) {
            assertTrue(pair.getValue().contentAsString().contains("line"));
            counter.incrementAndGet();
            String msg = pair.getValue().contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
            connectedServers.add(pair.getServer());
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            System.out.println("completed");
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    assertEquals("There should be as many as provided servers", localServerProvider.serverSize(), connectedServers.size());
    Assert.assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (ServerInfo server : localServerProvider.getServerInfos()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(server, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(server, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(server, EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals("There should be one completions", 1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}