Examples with Context io.mantisrx.runtime.Context used on opensource projects

Example 16 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class StageExecutorsGroupByTest method testExecuteSource.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSource() {
    TestGroupByJob provider = new TestGroupByJob();
    Job<Pair> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    int serverPort = portSelector.acquirePort();
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(serverPort, null, Observable.just(1), null);
    // execute source
    BehaviorSubject<Integer> workersInStageOneObservable = BehaviorSubject.create(1);
    StageExecutors.executeSource(0, job.getSource(), stages.get(0), producer, new Context(), workersInStageOneObservable);
    ConnectToGroupedObservable<String, Integer> config = new ConnectToGroupedObservable.Builder<String, Integer>().slotId("0").host("localhost").port(serverPort).keyDecoder(Codecs.string()).valueDecoder(Codecs.integer()).build();
    Iterator<GroupedObservable<String, Integer>> iter = RemoteObservable.connect(config).getObservable().toBlocking().getIterator();
    Assert.assertTrue(iter.hasNext());
    // verify numbers are grouped by even/odd
    // even is first due to zero
    GroupedObservable<String, Integer> even = iter.next();
    Assert.assertEquals("even", even.getKey());
    Iterator<Integer> evenIter = even.toBlocking().getIterator();
    Assert.assertEquals(0, evenIter.next().intValue());
    Assert.assertEquals(2, evenIter.next().intValue());
    Assert.assertEquals(4, evenIter.next().intValue());
    Assert.assertEquals(6, evenIter.next().intValue());
    GroupedObservable<String, Integer> odd = iter.next();
    Assert.assertEquals("odd", odd.getKey());
    Iterator<Integer> oddIter = odd.toBlocking().getIterator();
    Assert.assertEquals(1, oddIter.next().intValue());
    Assert.assertEquals(3, oddIter.next().intValue());
    Assert.assertEquals(5, oddIter.next().intValue());
    Assert.assertEquals(7, oddIter.next().intValue());
    // should only have two groups
    Assert.assertEquals(false, iter.hasNext());
}

Example 17 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class StageExecutorsGroupByTest method testExecuteIntermediatStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
    // Note, this test has a timing issue, client starts
    // sending data before server is ready, resulting
    // in a RST (connection reset by peer)
    TestGroupByJob provider = new TestGroupByJob();
    Job<Pair> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int publishPort = portSelector.acquirePort();
    final int consumerPort = portSelector.acquirePort();
    Observable<Observable<GroupedObservable<String, Integer>>> go = Observable.just(Observable.range(0, 10).groupBy(new Func1<Integer, String>() {

        @Override
        public String call(Integer t1) {
            if ((t1 % 2) == 0) {
                return "even";
            } else {
                return "odd";
            }
        }
    }));
    // mimic previous stage with a server
    ServeGroupedObservable<String, Integer> config = new ServeGroupedObservable.Builder<String, Integer>().keyEncoder(Codecs.string()).valueEncoder(Codecs.integer()).observable(go).build();
    RemoteRxServer server = new RemoteRxServer.Builder().addObservable(config).port(consumerPort).build();
    server.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "0")));
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
    // execute source
    StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
    ConnectToGroupedObservable<String, Integer> connectConfig = new ConnectToGroupedObservable.Builder<String, Integer>().host("localhost").port(publishPort).keyDecoder(Codecs.string()).valueDecoder(Codecs.integer()).build();
    Iterator<GroupedObservable<String, Integer>> iter = RemoteObservable.connect(connectConfig).getObservable().toBlocking().getIterator();
    // verify numbers are grouped by even/odd
    // even is first due to zero
    GroupedObservable<String, Integer> even = iter.next();
    Assert.assertEquals("even", even.getKey());
    Iterator<Integer> evenIter = even.toBlocking().getIterator();
    Assert.assertEquals(0, evenIter.next().intValue());
    Assert.assertEquals(4, evenIter.next().intValue());
    Assert.assertEquals(16, evenIter.next().intValue());
    Assert.assertEquals(36, evenIter.next().intValue());
    GroupedObservable<String, Integer> odd = iter.next();
    Assert.assertEquals("odd", odd.getKey());
    Iterator<Integer> oddIter = odd.toBlocking().getIterator();
    Assert.assertEquals(1, oddIter.next().intValue());
    Assert.assertEquals(9, oddIter.next().intValue());
    Assert.assertEquals(25, oddIter.next().intValue());
    Assert.assertEquals(49, oddIter.next().intValue());
    // should only have two groups
    Assert.assertEquals(false, iter.hasNext());
}

Example 18 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class StageExecutorsTest method testExecuteIntermediatStage.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
    TestJob provider = new TestJob();
    Job<Integer> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int publishPort = portSelector.acquirePort();
    final int consumerPort = portSelector.acquirePort();
    // mimic previous stage with a server
    RemoteRxServer server1 = RemoteObservable.serve(consumerPort, Observable.range(0, 10), Codecs.integer());
    server1.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
    // execute intermediate, flatten results
    StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
    Iterator<Integer> iter = RemoteObservable.connect(new ConnectToObservable.Builder<Integer>().host("localhost").slotId("0").port(publishPort).decoder(Codecs.integer()).build()).getObservable().toBlocking().getIterator();
    // verify numbers are even
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(2, iter.next().intValue());
    Assert.assertEquals(4, iter.next().intValue());
}

Example 19 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class StageExecutorsTest method testExecuteSink.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSink() throws InterruptedException {
    TestJob provider = new TestJob();
    Job<Integer> job = provider.getJobInstance();
    List<StageConfig<?, ?>> stages = job.getStages();
    PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
    final int consumerPort = portSelector.acquirePort();
    // mimic previous stage with a server
    RemoteRxServer server1 = RemoteObservable.serve(consumerPort, Observable.range(0, 10), Codecs.integer());
    server1.start();
    EndpointInjector staticEndpoints = new EndpointInjector() {

        @Override
        public Observable<EndpointChange> deltas() {
            return Observable.create(new OnSubscribe<EndpointChange>() {

                @Override
                public void call(Subscriber<? super EndpointChange> subscriber) {
                    subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
                    subscriber.onCompleted();
                }
            });
        }
    };
    Action0 noOpAction = new Action0() {

        @Override
        public void call() {
        }
    };
    Action1<Throwable> noOpError = new Action1<Throwable>() {

        @Override
        public void call(Throwable t) {
        }
    };
    WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
    // execute source
    StageExecutors.executeSink(consumer, stages.get(1), job.getSink(), new TestPortSelector(), new RxMetrics(), new Context(), noOpAction, null, null, noOpAction, noOpError);
    Iterator<Integer> iter = provider.getItemsWritten().iterator();
    // verify numbers are even
    Assert.assertEquals(0, iter.next().intValue());
    Assert.assertEquals(2, iter.next().intValue());
    Assert.assertEquals(4, iter.next().intValue());
}

Example 20 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class JobAutoScalerTest method testScaleDownNotLessThanMin.

@Test
public void testScaleDownNotLessThanMin() throws InterruptedException {
    final String jobId = "test-job-1";
    final int coolDownSec = 2;
    final int scalingStageNum = 1;
    final MantisMasterClientApi mockMasterClientApi = mock(MantisMasterClientApi.class);
    final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
    final int numStage1Workers = 5;
    final int increment = 10;
    // decrement by 10 on scale down, this will push num workers below min and below 0.
    final int decrement = 10;
    final int min = 3;
    final int max = 50;
    final double scaleUpAbovePct = 45.0;
    final double scaleDownBelowPct = 15.0;
    final double workerMemoryMB = 512.0;
    final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numStage1Workers).machineDefinition(new MachineDefinition(2, workerMemoryMB, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(scalingStageNum, min, max, increment, decrement, coolDownSec, Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, scaleDownBelowPct, scaleUpAbovePct, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
    schedulingInfoMap.put(scalingStageNum, stage1SchedInfo);
    when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), anyInt(), anyString())).thenReturn(Observable.just(true));
    Context context = mock(Context.class);
    when(context.getWorkerMapObservable()).thenReturn(Observable.empty());
    final JobAutoScaler jobAutoScaler = new JobAutoScaler(jobId, new SchedulingInfo(schedulingInfoMap), mockMasterClientApi, context);
    jobAutoScaler.start();
    final Observer<JobAutoScaler.Event> jobAutoScalerObserver = jobAutoScaler.getObserver();
    // should trigger a scale down (below 15% scaleDown threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleDownBelowPct / 100.0 - 0.01), numStage1Workers, ""));
    verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, min, String.format("Memory with value %1$,.2f is below scaleDown threshold of %2$,.1f", (scaleDownBelowPct / 100.0 - 0.01) * 100.0, scaleDownBelowPct));
    verifyNoMoreInteractions(mockMasterClientApi);
}