Examples with StringParameter io.mantisrx.runtime.parameter.type.StringParameter used on opensource projects

Example 1 with StringParameter

use of io.mantisrx.runtime.parameter.type.StringParameter in project mantis by Netflix.

the class KafkaSource method getParameters.

@Override
public List<ParameterDefinition<?>> getParameters() {
    final List<ParameterDefinition<?>> params = new ArrayList<>();
    params.add(new StringParameter().name(KafkaSourceParameters.TOPIC).description("Kafka topic to connect to").validator(Validators.notNullOrEmpty()).required().build());
    // Optional parameters
    params.add(new StringParameter().name(KafkaSourceParameters.CHECKPOINT_STRATEGY).description("checkpoint strategy one of " + CheckpointStrategyOptions.values() + " (ensure enable.auto.commit param is set to false when enabling this)").defaultValue(CheckpointStrategyOptions.NONE).validator(Validators.alwaysPass()).build());
    params.add(new IntParameter().name(KafkaSourceParameters.NUM_KAFKA_CONSUMER_PER_WORKER).description("No. of Kafka consumer instances per Mantis worker").validator(Validators.range(1, 16)).defaultValue(DEFAULT_NUM_KAFKA_CONSUMER_PER_WORKER).build());
    params.add(new IntParameter().name(KafkaSourceParameters.MAX_BYTES_IN_PROCESSING).description("The maximum amount of data per-consumer awaiting acks to trigger an offsets commit. " + "These commits are in addition to any commits triggered by commitIntervalMs timer").defaultValue(DEFAULT_MAX_BYTES_IN_PROCESSING).validator(Validators.range(1, Integer.MAX_VALUE)).build());
    params.add(new IntParameter().name(KafkaSourceParameters.CONSUMER_POLL_TIMEOUT_MS).validator(Validators.range(100, 10_000)).defaultValue(250).build());
    params.add(new StringParameter().name(KafkaSourceParameters.PARSER_TYPE).validator(Validators.notNullOrEmpty()).defaultValue(ParserType.SIMPLE_JSON.getPropName()).build());
    params.add(new BooleanParameter().name(KafkaSourceParameters.PARSE_MSG_IN_SOURCE).validator(Validators.alwaysPass()).defaultValue(DEFAULT_PARSE_MSG_IN_SOURCE).build());
    params.add(new BooleanParameter().name(KafkaSourceParameters.ENABLE_STATIC_PARTITION_ASSIGN).validator(Validators.alwaysPass()).defaultValue(DEFAULT_ENABLE_STATIC_PARTITION_ASSIGN).description("Disable Kafka's default consumer group management and statically assign partitions to job workers. When enabling static partition assignments, disable auto-scaling and set the numPartitionsPerTopic job parameter").build());
    params.add(new StringParameter().name(KafkaSourceParameters.TOPIC_PARTITION_COUNTS).validator(Validators.alwaysPass()).defaultValue("").description("Configures number of partitions on a kafka topic when static partition assignment is enabled. Format <topic1>:<numPartitions Topic1>,<topic2>:<numPartitions Topic2> Example: nf_errors_log:9,clevent:450").build());
    params.addAll(MantisKafkaConsumerConfig.getJobParameterDefinitions());
    return params;
}

Example 2 with StringParameter

use of io.mantisrx.runtime.parameter.type.StringParameter in project mantis by Netflix.

the class MantisKafkaConsumerConfig method getJobParameterDefinitions.

/**
 * Helper class to get all Kafka Consumer configs as Job Parameters to allow overriding Kafka consumer config settings at Job submit time.
 *
 * @return
 */
public static List<ParameterDefinition<?>> getJobParameterDefinitions() {
    List<ParameterDefinition<?>> params = new ArrayList<>();
    Map<String, Object> defaultProps = defaultProps();
    for (String key : configNames()) {
        ParameterDefinition.Builder<String> builder = new StringParameter().name(KafkaSourceParameters.PREFIX + key).validator(Validators.alwaysPass()).description(KafkaSourceParameters.PREFIX + key);
        if (defaultProps.containsKey(key)) {
            Object value = defaultProps.get(key);
            if (value instanceof Class) {
                builder = builder.defaultValue(((Class) value).getCanonicalName());
            } else {
                builder = builder.defaultValue((String) value);
            }
        }
        params.add(builder.build());
    }
    return params;
}

Example 3 with StringParameter

use of io.mantisrx.runtime.parameter.type.StringParameter in project mantis by Netflix.

the class TestJobParameterized method getJobInstance.

@Override
public Job<Integer> getJobInstance() {
    return MantisJob.<Integer>source(new Source<Integer>() {

        @Override
        public Observable<Observable<Integer>> call(Context context, Index index) {
            Integer start = (Integer) context.getParameters().get("start-range");
            Integer end = (Integer) context.getParameters().get("end-range");
            return Observable.just(Observable.range(start, end));
        }
    }).stage(new ScalarComputation<Integer, Integer>() {

        @Override
        public Observable<Integer> call(Context context, Observable<Integer> t1) {
            final Integer scale = (Integer) context.getParameters().get("scale-by");
            return t1.map(new Func1<Integer, Integer>() {

                @Override
                public Integer call(Integer t1) {
                    return t1 * scale;
                }
            });
        }
    }, new ScalarToScalar.Config<Integer, Integer>().codec(Codecs.integer())).sink(new Sink<Integer>() {

        @Override
        public void init(Context context) {
            System.out.println("sink init called");
        }

        @Override
        public void call(Context context, PortRequest p, Observable<Integer> o) {
            final String message = (String) context.getParameters().get("sink-message");
            o.toBlocking().forEach(new Action1<Integer>() {

                @Override
                public void call(Integer t1) {
                    System.out.println(message + t1);
                    itemsWritten.add(t1);
                }
            });
        }
    }).metadata(new Metadata.Builder().name("test job").description("showcase parameters").build()).parameterDefinition(new IntParameter().name("start-range").validator(Validators.range(0, 100)).build()).parameterDefinition(new IntParameter().name("end-range").validator(Validators.range(100, 1000)).build()).parameterDefinition(new IntParameter().name("scale-by").description("scale each value from the range").validator(Validators.range(1, 10)).build()).parameterDefinition(new StringParameter().name("sink-message").defaultValue("hello: ").description("concat with results").validator(Validators.notNullOrEmpty()).build()).create();
}

Example 4 with StringParameter

use of io.mantisrx.runtime.parameter.type.StringParameter in project mantis by Netflix.

the class ParameterDefinitionTest method singleRequiredParameterCheck.

@Test
public void singleRequiredParameterCheck() {
    Map<String, Object> parameterState = new HashMap<>();
    Map<String, ParameterDefinition<?>> parameterDefinitions = new HashMap<>();
    parameterDefinitions.put("foo", new StringParameter().name("foo").required().validator(Validators.<String>alwaysPass()).build());
    Map<String, Parameter> parameters = new HashMap<>();
    parameters.put("foo", new Parameter("foo", "test"));
    parameterState = ParameterUtils.checkThenCreateState(parameterDefinitions, parameters);
    Assert.assertEquals(1, parameterState.size());
    Assert.assertEquals(true, parameterState.containsKey("foo"));
    Assert.assertEquals("test", parameterState.get("foo"));
}

Example 5 with StringParameter

use of io.mantisrx.runtime.parameter.type.StringParameter in project mantis by Netflix.

the class ParameterDefinitionTest method singleRequiredSingleNonRequiredParameterCheck.

@Test
public void singleRequiredSingleNonRequiredParameterCheck() {
    Map<String, Object> parameterState = new HashMap<>();
    Map<String, ParameterDefinition<?>> parameterDefinitions = new HashMap<>();
    parameterDefinitions.put("required", new StringParameter().name("required").required().validator(Validators.<String>alwaysPass()).build());
    parameterDefinitions.put("nonRequired", new StringParameter().name("nonRequired").validator(Validators.<String>alwaysPass()).build());
    Map<String, Parameter> parameters = new HashMap<>();
    parameters.put("required", new Parameter("required", "test"));
    parameterState = ParameterUtils.checkThenCreateState(parameterDefinitions, parameters);
    Assert.assertEquals(1, parameterState.size());
    Assert.assertEquals(true, parameterState.containsKey("required"));
    Assert.assertEquals("test", parameterState.get("required"));
}