Examples with Context io.mantisrx.runtime.Context used on opensource projects

Example 36 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class KafkaSink method call.

@Override
public void call(Context context, PortRequest ignore, Observable<T> dataO) {
    if (kafkaProducerAtomicRef.get() == null) {
        MantisKafkaProducerConfig mantisKafkaProducerConfig = new MantisKafkaProducerConfig(context);
        Map<String, Object> producerProperties = mantisKafkaProducerConfig.getProducerProperties();
        KafkaProducer<byte[], byte[]> kafkaProducer = new KafkaProducer<>(producerProperties);
        kafkaProducerAtomicRef.compareAndSet(null, kafkaProducer);
        logger.info("Kafka Producer initialized");
    }
    KafkaProducer<byte[], byte[]> kafkaProducer = kafkaProducerAtomicRef.get();
    Parameters parameters = context.getParameters();
    String topic = (String) parameters.get(KafkaSinkJobParameters.TOPIC);
    dataO.map(encoder::call).flatMap((dataBytes) -> Observable.from(kafkaProducer.send(new ProducerRecord<>(topic, dataBytes))).subscribeOn(Schedulers.io())).subscribe();
}

Example 37 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class MantisKafkaProducerConfigTest method testJobParamOverrides.

@Test
public void testJobParamOverrides() {
    Context context = mock(Context.class);
    Parameters params = ParameterTestUtils.createParameters(KafkaSinkJobParameters.PREFIX + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaProducerConfig mantisKafkaProducerConfig = new MantisKafkaProducerConfig(context);
    Map<String, Object> producerProperties = mantisKafkaProducerConfig.getProducerProperties();
    assertEquals(StringSerializer.class, producerProperties.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG));
    assertEquals(ByteArraySerializer.class, producerProperties.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG));
}

Example 38 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class MantisKafkaProducerConfigTest method testDefaultKafkaProducerConfig.

@Test
public void testDefaultKafkaProducerConfig() {
    Context context = mock(Context.class);
    Parameters params = ParameterTestUtils.createParameters();
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaProducerConfig mantisKafkaProducerConfig = new MantisKafkaProducerConfig(context);
    Map<String, Object> producerProperties = mantisKafkaProducerConfig.getProducerProperties();
    assertEquals(ByteArraySerializer.class, producerProperties.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG));
    assertEquals(ByteArraySerializer.class, producerProperties.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG));
    assertEquals(Arrays.asList(MantisKafkaProducerConfig.DEFAULT_BOOTSTRAP_SERVERS_CONFIG), producerProperties.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG));
    assertEquals(Arrays.asList(JmxReporter.class.getName()), producerProperties.get(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG));
}

Example 39 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class MantisKafkaConsumerConfigTest method testDefaultConsumerConfig.

@Test
public void testDefaultConsumerConfig() {
    Context context = mock(Context.class);
    Parameters params = ParameterTestUtils.createParameters();
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaConsumerConfig mantisKafkaConsumerConfig = new MantisKafkaConsumerConfig(context);
    Map<String, Object> consumerProperties = mantisKafkaConsumerConfig.getConsumerProperties();
    assertEquals(Boolean.valueOf(MantisKafkaConsumerConfig.DEFAULT_AUTO_COMMIT_ENABLED), consumerProperties.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_AUTO_COMMIT_INTERVAL_MS, consumerProperties.get(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_AUTO_OFFSET_RESET, consumerProperties.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS, consumerProperties.get(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_FETCH_MIN_BYTES, consumerProperties.get(ConsumerConfig.FETCH_MIN_BYTES_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_HEARTBEAT_INTERVAL_MS, consumerProperties.get(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_SESSION_TIMEOUT_MS, consumerProperties.get(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_KEY_DESERIALIZER, consumerProperties.get(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_VALUE_DESERIALIZER, consumerProperties.get(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_MAX_PARTITION_FETCH_BYTES, consumerProperties.get(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_RECEIVE_BUFFER_BYTES, consumerProperties.get(ConsumerConfig.RECEIVE_BUFFER_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_SEND_BUFFER_BYTES, consumerProperties.get(ConsumerConfig.SEND_BUFFER_CONFIG));
    assertEquals(Arrays.asList(MantisKafkaConsumerConfig.DEFAULT_BOOTSTRAP_SERVERS_CONFIG), consumerProperties.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG));
    assertEquals(Arrays.asList(JmxReporter.class.getName()), consumerProperties.get(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG));
    assertEquals(Arrays.asList(RangeAssignor.class.getName()), consumerProperties.get(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.getGroupId(), consumerProperties.get(ConsumerConfig.GROUP_ID_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_MAX_POLL_INTERVAL_MS, consumerProperties.get(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_MAX_POLL_RECORDS, consumerProperties.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
    assertEquals(MantisKafkaConsumerConfig.DEFAULT_REQUEST_TIMEOUT_MS, consumerProperties.get(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG));
}

Example 40 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class JobAutoScalerTest method testScaleDown.

@Test
public void testScaleDown() throws InterruptedException {
    final String jobId = "test-job-1";
    final int coolDownSec = 2;
    final int scalingStageNum = 1;
    final MantisMasterClientApi mockMasterClientApi = mock(MantisMasterClientApi.class);
    final Map<Integer, StageSchedulingInfo> schedulingInfoMap = new HashMap<>();
    final int numStage1Workers = 2;
    final int increment = 1;
    final int decrement = 1;
    final int min = 1;
    final int max = 5;
    final double scaleUpAbovePct = 45.0;
    final double scaleDownBelowPct = 15.0;
    final double workerMemoryMB = 512.0;
    final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder().numberOfInstances(numStage1Workers).machineDefinition(new MachineDefinition(2, workerMemoryMB, 200, 1024, 2)).scalingPolicy(new StageScalingPolicy(scalingStageNum, min, max, increment, decrement, coolDownSec, Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, scaleDownBelowPct, scaleUpAbovePct, new StageScalingPolicy.RollingCount(1, 2))))).scalable(true).build();
    schedulingInfoMap.put(scalingStageNum, stage1SchedInfo);
    when(mockMasterClientApi.scaleJobStage(eq(jobId), eq(scalingStageNum), eq(numStage1Workers - decrement), anyString())).thenReturn(Observable.just(true));
    Context context = mock(Context.class);
    when(context.getWorkerMapObservable()).thenReturn(Observable.empty());
    final JobAutoScaler jobAutoScaler = new JobAutoScaler(jobId, new SchedulingInfo(schedulingInfoMap), mockMasterClientApi, context);
    jobAutoScaler.start();
    final Observer<JobAutoScaler.Event> jobAutoScalerObserver = jobAutoScaler.getObserver();
    // should trigger a scale down (below 15% scaleDown threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleDownBelowPct / 100.0 - 0.01), numStage1Workers, ""));
    verify(mockMasterClientApi, timeout(1000).times(1)).scaleJobStage(jobId, scalingStageNum, numStage1Workers - decrement, String.format("Memory with value %1$,.2f is below scaleDown threshold of %2$,.1f", (scaleDownBelowPct / 100.0 - 0.01) * 100.0, scaleDownBelowPct));
    // should *not* trigger a scale down before cooldown period (below 15% scaleDown threshold)
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleDownBelowPct / 100.0 - 0.01), numStage1Workers - decrement, ""));
    jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleDownBelowPct / 100.0 - 0.01), numStage1Workers - decrement, ""));
    Thread.sleep(coolDownSec * 1000);
    if (numStage1Workers - decrement == min) {
        // should not trigger a scale down after cooldown period if numWorkers=min (below 15% scaleDown threshold)
        jobAutoScalerObserver.onNext(new JobAutoScaler.Event(StageScalingPolicy.ScalingReason.Memory, scalingStageNum, workerMemoryMB * (scaleDownBelowPct / 100.0 - 0.01), numStage1Workers - decrement, ""));
        verifyNoMoreInteractions(mockMasterClientApi);
    }
}