Examples with ParameterDefinition io.mantisrx.runtime.parameter.ParameterDefinition used on opensource projects

Example 1 with ParameterDefinition

use of io.mantisrx.runtime.parameter.ParameterDefinition in project mantis by Netflix.

the class KafkaSource method getParameters.

@Override
public List<ParameterDefinition<?>> getParameters() {
    final List<ParameterDefinition<?>> params = new ArrayList<>();
    params.add(new StringParameter().name(KafkaSourceParameters.TOPIC).description("Kafka topic to connect to").validator(Validators.notNullOrEmpty()).required().build());
    // Optional parameters
    params.add(new StringParameter().name(KafkaSourceParameters.CHECKPOINT_STRATEGY).description("checkpoint strategy one of " + CheckpointStrategyOptions.values() + " (ensure enable.auto.commit param is set to false when enabling this)").defaultValue(CheckpointStrategyOptions.NONE).validator(Validators.alwaysPass()).build());
    params.add(new IntParameter().name(KafkaSourceParameters.NUM_KAFKA_CONSUMER_PER_WORKER).description("No. of Kafka consumer instances per Mantis worker").validator(Validators.range(1, 16)).defaultValue(DEFAULT_NUM_KAFKA_CONSUMER_PER_WORKER).build());
    params.add(new IntParameter().name(KafkaSourceParameters.MAX_BYTES_IN_PROCESSING).description("The maximum amount of data per-consumer awaiting acks to trigger an offsets commit. " + "These commits are in addition to any commits triggered by commitIntervalMs timer").defaultValue(DEFAULT_MAX_BYTES_IN_PROCESSING).validator(Validators.range(1, Integer.MAX_VALUE)).build());
    params.add(new IntParameter().name(KafkaSourceParameters.CONSUMER_POLL_TIMEOUT_MS).validator(Validators.range(100, 10_000)).defaultValue(250).build());
    params.add(new StringParameter().name(KafkaSourceParameters.PARSER_TYPE).validator(Validators.notNullOrEmpty()).defaultValue(ParserType.SIMPLE_JSON.getPropName()).build());
    params.add(new BooleanParameter().name(KafkaSourceParameters.PARSE_MSG_IN_SOURCE).validator(Validators.alwaysPass()).defaultValue(DEFAULT_PARSE_MSG_IN_SOURCE).build());
    params.add(new BooleanParameter().name(KafkaSourceParameters.ENABLE_STATIC_PARTITION_ASSIGN).validator(Validators.alwaysPass()).defaultValue(DEFAULT_ENABLE_STATIC_PARTITION_ASSIGN).description("Disable Kafka's default consumer group management and statically assign partitions to job workers. When enabling static partition assignments, disable auto-scaling and set the numPartitionsPerTopic job parameter").build());
    params.add(new StringParameter().name(KafkaSourceParameters.TOPIC_PARTITION_COUNTS).validator(Validators.alwaysPass()).defaultValue("").description("Configures number of partitions on a kafka topic when static partition assignment is enabled. Format <topic1>:<numPartitions Topic1>,<topic2>:<numPartitions Topic2> Example: nf_errors_log:9,clevent:450").build());
    params.addAll(MantisKafkaConsumerConfig.getJobParameterDefinitions());
    return params;
}

Example 2 with ParameterDefinition

use of io.mantisrx.runtime.parameter.ParameterDefinition in project mantis by Netflix.

the class MantisKafkaConsumerConfig method getJobParameterDefinitions.

/**
 * Helper class to get all Kafka Consumer configs as Job Parameters to allow overriding Kafka consumer config settings at Job submit time.
 *
 * @return
 */
public static List<ParameterDefinition<?>> getJobParameterDefinitions() {
    List<ParameterDefinition<?>> params = new ArrayList<>();
    Map<String, Object> defaultProps = defaultProps();
    for (String key : configNames()) {
        ParameterDefinition.Builder<String> builder = new StringParameter().name(KafkaSourceParameters.PREFIX + key).validator(Validators.alwaysPass()).description(KafkaSourceParameters.PREFIX + key);
        if (defaultProps.containsKey(key)) {
            Object value = defaultProps.get(key);
            if (value instanceof Class) {
                builder = builder.defaultValue(((Class) value).getCanonicalName());
            } else {
                builder = builder.defaultValue((String) value);
            }
        }
        params.add(builder.build());
    }
    return params;
}

Example 3 with ParameterDefinition

use of io.mantisrx.runtime.parameter.ParameterDefinition in project mantis by Netflix.

the class ParameterDefinitionTest method testEnumParameter.

@Test
public void testEnumParameter() {
    Map<String, ParameterDefinition<?>> parameterDefinitions = new HashMap<>();
    parameterDefinitions.put("foo", new EnumParameter<>(TestEnum.class).name("foo").required().validator(Validators.alwaysPass()).build());
    Map<String, Parameter> parameters = new HashMap<>();
    parameters.put("foo", new Parameter("foo", "A"));
    Map<String, Object> parameterState = ParameterUtils.checkThenCreateState(parameterDefinitions, parameters);
    Assert.assertEquals(TestEnum.A, parameterState.get("foo"));
}

Example 4 with ParameterDefinition

use of io.mantisrx.runtime.parameter.ParameterDefinition in project mantis by Netflix.

the class ParameterDefinitionTest method testEnumCSVParameter.

@Test
public void testEnumCSVParameter() {
    Map<String, ParameterDefinition<?>> parameterDefinitions = new HashMap<>();
    parameterDefinitions.put("foo", new EnumCSVParameter<>(TestEnum.class).name("foo").required().validator(Validators.alwaysPass()).build());
    Map<String, Parameter> parameters = new HashMap<>();
    parameters.put("foo", new Parameter("foo", "   A   ,  C   "));
    Map<String, Object> parameterState = ParameterUtils.checkThenCreateState(parameterDefinitions, parameters);
    EnumSet<TestEnum> foo = (EnumSet<TestEnum>) parameterState.get("foo");
    Assert.assertTrue(foo.contains(TestEnum.A));
    Assert.assertTrue(foo.contains(TestEnum.C));
    Assert.assertFalse(foo.contains(TestEnum.B));
}

Example 5 with ParameterDefinition

use of io.mantisrx.runtime.parameter.ParameterDefinition in project mantis by Netflix.

the class CreateJobDescriptorFile method createParameterInfo.

private Map<String, ParameterInfo> createParameterInfo(final Map<String, ParameterDefinition<?>> parameters) {
    final Map<String, ParameterInfo> parameterInfo = new HashMap<>();
    for (Entry<String, ParameterDefinition<?>> entry : parameters.entrySet()) {
        ParameterDefinition<?> definition = entry.getValue();
        String defaultValue = null;
        if (definition.getDefaultValue() != null) {
            defaultValue = definition.getDefaultValue().toString();
        }
        parameterInfo.put(entry.getKey(), new ParameterInfo(definition.getName(), definition.getDescription(), defaultValue, definition.getTypeDescription(), definition.getValidator().getDescription(), definition.isRequired()));
    }
    return parameterInfo;
}