Examples with Context io.mantisrx.runtime.Context used on opensource projects

Example 41 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class JobAutoScalerTest method testScaleUp.

@Test
public void testScaleUp() throws InterruptedException {
    final String jobId = "test-job-1";
    final int coolDownSec = 2;
    final int scalingStageNum = 1;
    final MantisMasterClientApi mockMasterClientApi = mock(MantisMasterClientApi.class);
    final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
    final int numStage1Workers = 1;
    final int increment = 1;
    final int decrement = 1;
    final int min = 1;
    final int max = 5;
    final double scaleUpAbovePct = 45.0;
    final double scaleDownBelowPct = 15.0;
    final double workerMemoryMB = 512.0;
    final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numStage1Workers).machineDefinition(new MachineDefinition(2, workerMemoryMB, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(scalingStageNum, min, max, increment, decrement, coolDownSec, Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, scaleDownBelowPct, scaleUpAbovePct, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
    schedulingInfoMap.put(scalingStageNum, stage1SchedInfo);
    when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), eq(numStage1Workers + increment), anyString())).thenReturn(Observable.just(true));
    Context context = mock(Context.class);
    when(context.getWorkerMapObservable()).thenReturn(Observable.empty());
    final JobAutoScaler jobAutoScaler = new JobAutoScaler(jobId, new SchedulingInfo(schedulingInfoMap), mockMasterClientApi, context);
    jobAutoScaler.start();
    final Observer<JobAutoScaler.Event> jobAutoScalerObserver = jobAutoScaler.getObserver();
    // should trigger a scale up (above 45% scaleUp threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleUpAbovePct / 100.0 + 0.01), numStage1Workers, ""));
    verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, numStage1Workers + increment, String.format("Memory with value %1$,.2f exceeded scaleUp threshold of 45.0", (scaleUpAbovePct / 100.0 + 0.01) * 100.0));
    // should *not* trigger a scale up before cooldown period (above 45% scaleUp threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleUpAbovePct / 100.0 + 0.01), numStage1Workers + increment, ""));
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleUpAbovePct / 100.0 + 0.01), numStage1Workers + increment, ""));
    Thread.sleep(coolDownSec * 1000);
    // retry sending auto scale event till scaleJobStage request sent to master, as there is possible a race between the sleep for coolDownSecs in the Test and the event being processed before coolDownSecs
    final CountDownLatch retryLatch = new CountDownLatch(1);
    when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), eq(numStage1Workers + 2 * increment), anyString())).thenAnswer(new Answer<Observable<Void>>() {

        @Override
        public Observable<Void> answer(InvocationOnMock invocation) throws Throwable {
            retryLatch.countDown();
            return Observable.just(null);
        }
    });
    do {
        logger.info("sending Job auto scale Event");
        // should trigger a scale up after cooldown period (above 45% scaleUp threshold)
        jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleUpAbovePct / 100.0 + 0.01), numStage1Workers + increment, ""));
    } while (!retryLatch.await(1, TimeUnit.SECONDS));
    verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, numStage1Workers + 2 * increment, String.format("Memory with value %1$,.2f exceeded scaleUp threshold of 45.0", (scaleUpAbovePct / 100.0 + 0.01) * 100.0));
}

Example 42 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class IcebergWriterStageTest method setUp.

@BeforeEach
void setUp() {
    record = GenericRecord.create(SCHEMA);
    record.setField("id", 1);
    this.scheduler = new TestScheduler();
    this.subscriber = new TestSubscriber<>();
    // Writer
    Parameters parameters = StageOverrideParameters.newParameters();
    WriterConfig config = new WriterConfig(parameters, mock(Configuration.class));
    WriterMetrics metrics = new WriterMetrics();
    IcebergWriterFactory factory = FakeIcebergWriter::new;
    this.writerPool = spy(new FixedIcebergWriterPool(factory, config.getWriterFlushFrequencyBytes(), config.getWriterMaximumPoolSize()));
    doReturn(Collections.singleton(record)).when(writerPool).getFlushableWriters();
    this.partitioner = mock(Partitioner.class);
    when(partitioner.partition(record)).thenReturn(record);
    this.transformer = new IcebergWriterStage.Transformer(config, metrics, this.writerPool, this.partitioner, this.scheduler, this.scheduler);
    // Catalog
    ServiceLocator serviceLocator = mock(ServiceLocator.class);
    when(serviceLocator.service(Configuration.class)).thenReturn(mock(Configuration.class));
    this.catalog = mock(Catalog.class);
    Table table = mock(Table.class);
    PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("id").build();
    when(table.spec()).thenReturn(spec);
    when(this.catalog.loadTable(any())).thenReturn(table);
    when(serviceLocator.service(Catalog.class)).thenReturn(this.catalog);
    when(serviceLocator.service(PartitionerFactory.class)).thenReturn(mock(PartitionerFactory.class));
    // Mantis Context
    this.context = mock(Context.class);
    when(this.context.getParameters()).thenReturn(parameters);
    when(this.context.getServiceLocator()).thenReturn(serviceLocator);
    when(this.context.getWorkerInfo()).thenReturn(new WorkerInfo("testJobName", "jobId", 1, 1, 1, MantisJobDurationType.Perpetual, "host"));
    // Flow
    Observable<Record> source = Observable.interval(1, TimeUnit.MILLISECONDS, this.scheduler).map(i -> record);
    this.flow = source.compose(this.transformer);
}

Example 43 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class HttpSourceImplTest method testGettingStreamFromMultipleServers.

@Test
public void testGettingStreamFromMultipleServers() throws Exception {
    HttpSourceImpl<ByteBuf, ServerSentEvent, ServerContext<ServerSentEvent>> source = createStreamingSource();
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    final ConcurrentHashMap<String, AtomicInteger> result = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).doOnNext(new Action1<ServerContext<ServerSentEvent>>() {

        @Override
        public void call(ServerContext<ServerSentEvent> pair) {
            assertTrue(pair.getValue().contentAsString().contains("line"));
            counter.incrementAndGet();
            String msg = pair.getValue().contentAsString();
            result.putIfAbsent(msg, new AtomicInteger());
            result.get(msg).incrementAndGet();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    long waitSeconds = 3;
    boolean timedout = !done.await(waitSeconds, TimeUnit.SECONDS);
    if (timedout) {
        fail(String.format("Waited at least %d seconds for the test to finish. Something is wrong", waitSeconds));
    }
    assertEquals(String.format("%d servers => the result has %d times of a single stream", localServerProvider.serverSize(), localServerProvider.serverSize()), counter.get(), RequestProcessor.smallStreamContent.size() * localServerProvider.serverSize());
    for (String data : RequestProcessor.smallStreamContent) {
        assertEquals(String.format("%d servers => %d identical copies per message", localServerProvider.serverSize(), localServerProvider.serverSize()), localServerProvider.serverSize(), result.get(data).get());
    }
    for (Server server : localServerProvider.getServers()) {
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(toServerInfo(server), EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(toServerInfo(server), EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(toServerInfo(server), EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(toServerInfo(server), EventType.CONNECTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
    assertEquals("completed source should clean up its retry servers", 0, source.getRetryServers().size());
}

Example 44 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class HttpSourceImplTest method testStreamingErrorFromServerWillNotCompleteSource.

@Test
public void testStreamingErrorFromServerWillNotCompleteSource() throws Exception {
    int eventCount = 20;
    HttpSourceImpl<ByteBuf, ServerSentEvent, ServerContext<ServerSentEvent>> source = createStreamingSource("test/finiteStream?count=" + eventCount);
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicReference<Throwable> ex = new AtomicReference<>();
    final ConcurrentHashMap<ServerInfo, Set<String>> items = new ConcurrentHashMap<>();
    Observable.merge(source.call(new Context(), new Index(1, 1))).subscribe(new Subscriber<ServerContext<ServerSentEvent>>() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public void onError(Throwable e) {
            ex.set(e);
            latch.countDown();
        }

        @Override
        public void onNext(ServerContext<ServerSentEvent> pair) {
            items.putIfAbsent(pair.getServer(), new HashSet<String>());
            items.get(pair.getServer()).add(pair.getValue().contentAsString());
        }
    });
    if (latch.await(5, TimeUnit.SECONDS)) {
        fail("The test case should not finish at all");
    }
    Assert.assertNull("The timeout error should be captured by the client so it does not surface to the source", ex.get());
    for (Server server : localServerProvider.getServers()) {
        ServerInfo serverInfo = toServerInfo(server);
        assertEquals("There should be no source level error", 0, sourceObserver.getErrorCount());
        assertEquals("There should be one connection attempt per server", 1, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ATTEMPTED));
        assertEquals("There should be one established connection per server ", 1, sourceObserver.getCount(serverInfo, EventType.CONNECTION_ESTABLISHED));
        assertEquals("There should no subscribed server because of read timeout", 1, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
        assertEquals(String.format("There should be %d item before simulated error per server", eventCount), eventCount, items.get(serverInfo).size());
    }
}

Example 45 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class HttpSourceImplTest method testGettingSingleEntityFromMultipleServers.

@Test
public void testGettingSingleEntityFromMultipleServers() throws Exception {
    HttpSourceImpl<ByteBuf, ByteBuf, ServerContext<ByteBuf>> source = createSingleEntitySource();
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch done = new CountDownLatch(1);
    Observable.merge(source.call(new Context(), new Index(1, 1))).map(new Func1<ServerContext<ByteBuf>, ServerContext<String>>() {

        @Override
        public ServerContext<String> call(ServerContext<ByteBuf> pair) {
            return new ServerContext<>(pair.getServer(), pair.getValue().toString(Charset.defaultCharset()));
        }
    }).doOnNext(new Action1<ServerContext<String>>() {

        @Override
        public void call(ServerContext<String> pair) {
            counter.incrementAndGet();
            assertEquals(RequestProcessor.SINGLE_ENTITY_RESPONSE, pair.getValue());
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("Unexpected failure: " + throwable);
        }
    }).doAfterTerminate(new Action0() {

        @Override
        public void call() {
            done.countDown();
        }
    }).subscribe();
    done.await(3000, TimeUnit.SECONDS);
    assertEquals(String.format("There should be exactly one response from each of the %d servers", localServerProvider.serverSize()), localServerProvider.serverSize(), counter.get());
    for (Server server : localServerProvider.getServers()) {
        ServerInfo serverInfo = toServerInfo(server);
        assertEquals("There should be one completion per server", 1, sourceObserver.getCount(serverInfo, EventType.SOURCE_COMPLETED));
        assertEquals("There should be one un-subscription per server", 1, sourceObserver.getCount(serverInfo, EventType.CONNECTION_UNSUBSCRIBED));
        assertEquals("There should be no error", 0, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_FAILED));
        assertEquals("There should be one connection per server", 1, sourceObserver.getCount(serverInfo, EventType.SUBSCRIPTION_ESTABLISHED));
    }
    assertEquals(1, sourceObserver.getCompletionCount());
    assertEquals(0, sourceObserver.getErrorCount());
    Set<EventType> events = sourceObserver.getEvents();
    assertEquals(EXPECTED_EVENTS_SETS, events);
    for (EventType event : events) {
        assertEquals("Each event should be recorded exactly once per server", localServerProvider.serverSize(), sourceObserver.getEventCount(event));
    }
}