Examples with ConsumerConfig org.apache.pulsar.common.functions.ConsumerConfig used on opensource projects

Example 1 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class FunctionConfigUtilsTest method testMergeDifferentInputSpecWithRegexChange.

@Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "isRegexPattern for input topic test-input cannot be altered")
public void testMergeDifferentInputSpecWithRegexChange() {
    FunctionConfig functionConfig = createFunctionConfig();
    Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
    inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(false).serdeClassName("my-serde").build());
    FunctionConfig newFunctionConfig = createUpdatedFunctionConfig("inputSpecs", inputSpecs);
    FunctionConfigUtils.validateUpdate(functionConfig, newFunctionConfig);
}

Example 2 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class FunctionConfigUtilsTest method createFunctionConfig.

private FunctionConfig createFunctionConfig() {
    FunctionConfig functionConfig = new FunctionConfig();
    functionConfig.setTenant("test-tenant");
    functionConfig.setNamespace("test-namespace");
    functionConfig.setName("test-function");
    functionConfig.setParallelism(1);
    functionConfig.setClassName(IdentityFunction.class.getName());
    Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
    inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build());
    functionConfig.setInputSpecs(inputSpecs);
    functionConfig.setOutput("test-output");
    functionConfig.setOutputSerdeClassName("test-serde");
    functionConfig.setOutputSchemaType("json");
    functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setRetainOrdering(false);
    functionConfig.setRetainKeyOrdering(false);
    functionConfig.setSubscriptionPosition(SubscriptionInitialPosition.Earliest);
    functionConfig.setBatchBuilder("DEFAULT");
    functionConfig.setForwardSourceMessageProperty(false);
    functionConfig.setUserConfig(new HashMap<>());
    functionConfig.setAutoAck(true);
    functionConfig.setTimeoutMs(2000l);
    functionConfig.setWindowConfig(new WindowConfig().setWindowLengthCount(10));
    functionConfig.setCleanupSubscription(true);
    functionConfig.setRuntimeFlags("-Dfoo=bar");
    return functionConfig;
}

Example 3 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class FunctionConfigUtilsTest method testConvertBackFidelity.

@Test
public void testConvertBackFidelity() {
    FunctionConfig functionConfig = new FunctionConfig();
    functionConfig.setTenant("test-tenant");
    functionConfig.setNamespace("test-namespace");
    functionConfig.setName("test-function");
    functionConfig.setParallelism(1);
    functionConfig.setClassName(IdentityFunction.class.getName());
    Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
    inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").poolMessages(true).build());
    functionConfig.setInputSpecs(inputSpecs);
    functionConfig.setOutput("test-output");
    functionConfig.setOutputSerdeClassName("test-serde");
    functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setRetainOrdering(false);
    functionConfig.setRetainKeyOrdering(false);
    functionConfig.setForwardSourceMessageProperty(true);
    functionConfig.setUserConfig(new HashMap<>());
    functionConfig.setAutoAck(true);
    functionConfig.setTimeoutMs(2000l);
    functionConfig.setRuntimeFlags("-DKerberos");
    ProducerConfig producerConfig = new ProducerConfig();
    producerConfig.setMaxPendingMessages(100);
    producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
    producerConfig.setUseThreadLocalProducers(true);
    producerConfig.setBatchBuilder("DEFAULT");
    functionConfig.setProducerConfig(producerConfig);
    Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, null);
    FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
    // add default resources
    functionConfig.setResources(Resources.getDefaultResources());
    // set default cleanupSubscription config
    functionConfig.setCleanupSubscription(true);
    assertEquals(new Gson().toJson(functionConfig), new Gson().toJson(convertedConfig));
}

Example 4 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class SinkConfigUtilsTest method testConvertBackFidelity.

@Test
public void testConvertBackFidelity() throws IOException {
    SinkConfig sinkConfig = new SinkConfig();
    sinkConfig.setTenant("test-tenant");
    sinkConfig.setNamespace("test-namespace");
    sinkConfig.setName("test-source");
    sinkConfig.setParallelism(1);
    sinkConfig.setArchive("builtin://jdbc");
    sinkConfig.setSourceSubscriptionName("test-subscription");
    Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
    inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).receiverQueueSize(532).serdeClassName("test-serde").poolMessages(true).build());
    sinkConfig.setInputs(Collections.singleton("test-input"));
    sinkConfig.setInputSpecs(inputSpecs);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    Map<String, String> producerConfigs = new HashMap<>();
    producerConfigs.put("security.protocal", "SASL_PLAINTEXT");
    Map<String, Object> configs = new HashMap<>();
    configs.put("topic", "kafka");
    configs.put("bootstrapServers", "server-1,server-2");
    configs.put("producerConfigProperties", producerConfigs);
    sinkConfig.setConfigs(configs);
    sinkConfig.setRetainOrdering(false);
    sinkConfig.setRetainKeyOrdering(false);
    sinkConfig.setAutoAck(true);
    sinkConfig.setCleanupSubscription(false);
    sinkConfig.setTimeoutMs(2000l);
    sinkConfig.setRuntimeFlags("-DKerberos");
    sinkConfig.setCleanupSubscription(true);
    sinkConfig.setResources(Resources.getDefaultResources());
    Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
    assertEquals(Function.SubscriptionType.SHARED, functionDetails.getSource().getSubscriptionType());
    SinkConfig convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails);
    assertEquals(new Gson().toJson(convertedConfig), new Gson().toJson(sinkConfig));
    sinkConfig.setRetainOrdering(true);
    sinkConfig.setRetainKeyOrdering(false);
    functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
    assertEquals(Function.SubscriptionType.FAILOVER, functionDetails.getSource().getSubscriptionType());
    convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails);
    assertEquals(new Gson().toJson(convertedConfig), new Gson().toJson(sinkConfig));
    sinkConfig.setRetainOrdering(false);
    sinkConfig.setRetainKeyOrdering(true);
    functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
    assertEquals(Function.SubscriptionType.KEY_SHARED, functionDetails.getSource().getSubscriptionType());
    convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails);
    assertEquals(new Gson().toJson(convertedConfig), new Gson().toJson(sinkConfig));
}

Example 5 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class SinkConfigUtilsTest method testPoolMessages.

@Test
public void testPoolMessages() throws IOException {
    SinkConfig sinkConfig = createSinkConfig();
    Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
    assertFalse(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages());
    SinkConfig convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails);
    assertFalse(convertedConfig.getInputSpecs().get("test-input").isPoolMessages());
    Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
    inputSpecs.put("test-input", ConsumerConfig.builder().poolMessages(true).build());
    sinkConfig.setInputSpecs(inputSpecs);
    functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null));
    assertTrue(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages());
    convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails);
    assertTrue(convertedConfig.getInputSpecs().get("test-input").isPoolMessages());
}