Examples with Parameters io.mantisrx.runtime.parameter.Parameters used on opensource projects

Example 1 with Parameters

use of io.mantisrx.runtime.parameter.Parameters in project mantis by Netflix.

the class MantisKafkaConsumerConfigTest method testJobParamOverrides.

@Test
public void testJobParamOverrides() {
    Context context = mock(Context.class);
    String testTopic = "topic123";
    String testConsumerGroupId = "testKafkaConsumer-1";
    Parameters params = ParameterTestUtils.createParameters(KafkaSourceParameters.TOPIC, testTopic, KafkaSourceParameters.PREFIX + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest", KafkaSourceParameters.PREFIX + ConsumerConfig.GROUP_ID_CONFIG, testConsumerGroupId, KafkaSourceParameters.PREFIX + ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 500);
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaConsumerConfig mantisKafkaConsumerConfig = new MantisKafkaConsumerConfig(context);
    Map<String, Object> consumerProperties = mantisKafkaConsumerConfig.getConsumerProperties();
    // MantisKafkaConsumerConfig only affects Kafka's ConsumerConfig defined properties
    assertFalse(ConsumerConfig.configNames().contains(KafkaSourceParameters.TOPIC));
    assertFalse(consumerProperties.containsKey(KafkaSourceParameters.TOPIC));
    assertEquals("earliest", consumerProperties.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
    assertEquals(testConsumerGroupId, consumerProperties.get(ConsumerConfig.GROUP_ID_CONFIG));
    assertEquals(500, consumerProperties.get(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG));
}

Example 2 with Parameters

use of io.mantisrx.runtime.parameter.Parameters in project mantis by Netflix.

the class MantisKafkaConsumerConfig method applyJobParamOverrides.

private static Map<String, Object> applyJobParamOverrides(Context context, Map<String, Object> parsedValues) {
    final Parameters parameters = context.getParameters();
    if (!parsedValues.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
        // set consumerGroupId if not already set
        final String consumerGroupId = (String) parameters.get(KafkaSourceParameters.PREFIX + ConsumerConfig.GROUP_ID_CONFIG, getGroupId());
        parsedValues.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId);
    }
    for (String key : configNames()) {
        Object value = parameters.get(KafkaSourceParameters.PREFIX + key, null);
        if (value != null) {
            LOGGER.info("job param override for key {} -> {}", key, value);
            parsedValues.put(key, value);
        }
    }
    return parsedValues;
}

Example 3 with Parameters

use of io.mantisrx.runtime.parameter.Parameters in project mantis by Netflix.

the class MantisKafkaSourceConfigTest method testJobParamOverrides.

@Test
public void testJobParamOverrides() {
    Context context = mock(Context.class);
    String testTopic = "topic123";
    int checkpointIntervalOverride = 100;
    boolean staticPartitionAssignEnableOverride = true;
    Parameters params = ParameterTestUtils.createParameters(KafkaSourceParameters.TOPIC, testTopic, KafkaSourceParameters.CHECKPOINT_INTERVAL_MS, checkpointIntervalOverride, KafkaSourceParameters.ENABLE_STATIC_PARTITION_ASSIGN, staticPartitionAssignEnableOverride, KafkaSourceParameters.TOPIC_PARTITION_COUNTS, testTopic + ":1024");
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    MantisKafkaSourceConfig mantisKafkaSourceConfig = new MantisKafkaSourceConfig(context);
    assertEquals(checkpointIntervalOverride, mantisKafkaSourceConfig.getCheckpointIntervalMs());
    assertEquals(staticPartitionAssignEnableOverride, mantisKafkaSourceConfig.getStaticPartitionAssignmentEnabled());
    Map<String, Integer> topicPartitionCounts = new HashMap<>();
    topicPartitionCounts.put(testTopic, 1024);
    assertEquals(Optional.ofNullable(topicPartitionCounts), mantisKafkaSourceConfig.getTopicPartitionCounts());
}

Example 4 with Parameters

use of io.mantisrx.runtime.parameter.Parameters in project mantis by Netflix.

the class IcebergCommitterStageTest method setUp.

@BeforeEach
void setUp() {
    this.scheduler = new TestScheduler();
    this.subscriber = new TestSubscriber<>();
    Parameters parameters = StageOverrideParameters.newParameters();
    CommitterConfig config = new CommitterConfig(parameters);
    CommitterMetrics metrics = new CommitterMetrics();
    this.committer = mock(IcebergCommitter.class);
    transformer = new IcebergCommitterStage.Transformer(config, metrics, committer, scheduler);
    ServiceLocator serviceLocator = mock(ServiceLocator.class);
    when(serviceLocator.service(Configuration.class)).thenReturn(mock(Configuration.class));
    this.catalog = mock(Catalog.class);
    Table table = mock(Table.class);
    when(table.spec()).thenReturn(PartitionSpec.unpartitioned());
    when(this.catalog.loadTable(any())).thenReturn(table);
    when(serviceLocator.service(Catalog.class)).thenReturn(this.catalog);
    this.context = mock(Context.class);
    when(this.context.getParameters()).thenReturn(parameters);
    when(this.context.getServiceLocator()).thenReturn(serviceLocator);
}

Example 5 with Parameters

use of io.mantisrx.runtime.parameter.Parameters in project mantis by Netflix.

the class KafkaSinkTest method testKafkaSink.

@Test
public void testKafkaSink() throws InterruptedException {
    String testTopic = "testTopic" + topicNum.incrementAndGet();
    int numPartitions = 1;
    kafkaServer.createTopic(testTopic, numPartitions);
    int numMessages = 10;
    KafkaSink<String> kafkaSink = new KafkaSink<>(new NoopRegistry(), s -> s.getBytes());
    Context context = mock(Context.class);
    Parameters params = ParameterTestUtils.createParameters(KafkaSinkJobParameters.TOPIC, testTopic);
    when(context.getParameters()).then((Answer<Parameters>) invocation -> params);
    when(context.getWorkerInfo()).then((Answer<WorkerInfo>) invocation -> new WorkerInfo("testJobName", "testJobName-1", 1, 0, 1, MantisJobDurationType.Perpetual, "1.1.1.1"));
    when(context.getJobId()).then((Answer<String>) invocation -> "testJobName-1");
    kafkaSink.call(context, mock(PortRequest.class), Observable.range(0, numMessages).map(x -> String.valueOf(x)));
    List<String> messages = kafkaServer.readAllMessages(testTopic);
    LOGGER.info("got {}", messages);
    assertEquals(numMessages, messages.size());
    for (int i = 0; i < numMessages; i++) {
        assertEquals(i, Integer.parseInt(messages.get(i)));
    }
    kafkaServer.deleteTopic(testTopic);
}