Examples with Context io.mantisrx.runtime.Context used on opensource projects

Example 1 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class MantisKafkaConsumerConfigTest method testJobParamOverrides.

@Test
public void testJobParamOverrides() {
    Context context = mock(Context.class);
    String testTopic = "topic123";
    String testConsumerGroupId = "testKafkaConsumer-1";
    Parameters params = ParameterTestUtils.createParameters(KafkaSourceParameters.TOPIC, testTopic, KafkaSourceParameters.PREFIX + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest", KafkaSourceParameters.PREFIX + ConsumerConfig.GROUP_ID_CONFIG, testConsumerGroupId, KafkaSourceParameters.PREFIX + ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 500);
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaConsumerConfig mantisKafkaConsumerConfig = new MantisKafkaConsumerConfig(context);
    Map<String, Object> consumerProperties = mantisKafkaConsumerConfig.getConsumerProperties();
    // MantisKafkaConsumerConfig only affects Kafka's ConsumerConfig defined properties
    assertFalse(ConsumerConfig.configNames().contains(KafkaSourceParameters.TOPIC));
    assertFalse(consumerProperties.containsKey(KafkaSourceParameters.TOPIC));
    assertEquals("earliest", consumerProperties.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
    assertEquals(testConsumerGroupId, consumerProperties.get(ConsumerConfig.GROUP_ID_CONFIG));
    assertEquals(500, consumerProperties.get(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG));
}

Example 2 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class MantisKafkaSourceConfigTest method testJobParamOverrides.

@Test
public void testJobParamOverrides() {
    Context context = mock(Context.class);
    String testTopic = "topic123";
    int checkpointIntervalOverride = 100;
    boolean staticPartitionAssignEnableOverride = true;
    Parameters params = ParameterTestUtils.createParameters(KafkaSourceParameters.TOPIC, testTopic, KafkaSourceParameters.CHECKPOINT_INTERVAL_MS, checkpointIntervalOverride, KafkaSourceParameters.ENABLE_STATIC_PARTITION_ASSIGN, staticPartitionAssignEnableOverride, KafkaSourceParameters.TOPIC_PARTITION_COUNTS, testTopic + ":1024");
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaSourceConfig mantisKafkaSourceConfig = new MantisKafkaSourceConfig(context);
    assertEquals(checkpointIntervalOverride, mantisKafkaSourceConfig.getCheckpointIntervalMs());
    assertEquals(staticPartitionAssignEnableOverride, mantisKafkaSourceConfig.getStaticPartitionAssignmentEnabled());
    Map<String, Integer> topicPartitionCounts = new HashMap<>();
    topicPartitionCounts.put(testTopic, 1024);
    assertEquals(Optional.ofNullable(topicPartitionCounts), mantisKafkaSourceConfig.getTopicPartitionCounts());
}

Example 3 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class KafkaSinkTest method testKafkaSink.

@Test
public void testKafkaSink() throws InterruptedException {
    String testTopic = "testTopic" + topicNum.incrementAndGet();
    int numPartitions = 1;
    kafkaServer.createTopic(testTopic, numPartitions);
    int numMessages = 10;
    KafkaSink<String> kafkaSink = new KafkaSink<>(new NoopRegistry(), s -> s.getBytes());
    Context context = mock(Context.class);
    Parameters params = ParameterTestUtils.createParameters(KafkaSinkJobParameters.TOPIC, testTopic);
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    when(context.getWorkerInfo()).then((Answer<WorkerInfo>) invocation -> new WorkerInfo("testJobName", "testJobName-1", 1, 0, 1, MantisJobDurationType.Perpetual, "1.1.1.1"));
    when(context.getJobId()).then((Answer<String>) invocation -> "testJobName-1");
    kafkaSink.call(context, mock(PortRequest.class), Observable.range(0, numMessages).map(x -> String.valueOf(x)));
    List<String> messages = kafkaServer.readAllMessages(testTopic);
    LOGGER.info("got {}", messages);
    assertEquals(numMessages, messages.size());
    for (int i = 0; i < numMessages; i++) {
        assertEquals(i, Integer.parseInt(messages.get(i)));
    }
    kafkaServer.deleteTopic(testTopic);
}

Example 4 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class KafkaSourceTest method testKafkaSourceSingleConsumerReadsAllMessagesInOrderFromSinglePartition.

@Test
public void testKafkaSourceSingleConsumerReadsAllMessagesInOrderFromSinglePartition() throws InterruptedException {
    String testTopic = "testTopic" + topicNum.incrementAndGet();
    int numPartitions = 1;
    kafkaServer.createTopic(testTopic, numPartitions);
    int numMessages = 10;
    for (int i = 0; i < numMessages; i++) {
        ProducerRecord<String, String> keyedMessage = new ProducerRecord<>(testTopic, "{\"messageNum\":" + i + "}");
        kafkaServer.sendMessages(keyedMessage);
    }
    KafkaSource kafkaSource = new KafkaSource(new NoopRegistry());
    Context context = mock(Context.class);
    Parameters params = ParameterTestUtils.createParameters(KafkaSourceParameters.TOPIC, testTopic, KafkaSourceParameters.PREFIX + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest", KafkaSourceParameters.PREFIX + ConsumerConfig.GROUP_ID_CONFIG, "testKafkaConsumer-" + random.nextInt());
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    when(context.getWorkerInfo()).then((Answer<WorkerInfo>) invocation -> new WorkerInfo("testJobName", "testJobName-1", 1, 0, 1, MantisJobDurationType.Perpetual, "1.1.1.1"));
    when(context.getJobId()).then((Answer<String>) invocation -> "testJobName-1");
    Index index = new Index(0, 10);
    Observable<Observable<KafkaAckable>> sourceObs = kafkaSource.call(context, index);
    final CountDownLatch latch = new CountDownLatch(numMessages);
    final AtomicInteger counter = new AtomicInteger(0);
    sourceObs.flatMap(kafkaAckableObs -> kafkaAckableObs).map(kafkaAckable -> {
        Optional<Map<String, Object>> parsedEvent = kafkaAckable.getKafkaData().getParsedEvent();
        assertTrue(parsedEvent.isPresent());
        assertEquals(counter.getAndIncrement(), parsedEvent.get().get("messageNum"));
        LOGGER.info("got message on topic {} consumer Id {}", parsedEvent.get(), kafkaAckable.getKafkaData().getMantisKafkaConsumerId());
        kafkaAckable.ack();
        latch.countDown();
        return parsedEvent;
    }).subscribe();
    assertTrue("timed out waiting to get all messages from Kafka", latch.await(10, TimeUnit.SECONDS));
    kafkaServer.deleteTopic(testTopic);
}

Example 5 with Context

use of io.mantisrx.runtime.Context in project mantis by Netflix.

the class SinePointGeneratorStage method call.

@Override
public Observable<Point> call(Context context, Observable<Integer> o) {
    final double amplitude = (double) context.getParameters().get(SineFunctionJob.AMPLITUDE);
    final double frequency = (double) context.getParameters().get(SineFunctionJob.FREQUENCY);
    final double phase = (double) context.getParameters().get(SineFunctionJob.PHASE);
    return o.filter(x -> x % 2 == 0).map(x -> new Point(x, amplitude * Math.sin((frequency * x) + phase)));
}