Examples with ConsumerConfig org.apache.pulsar.common.functions.ConsumerConfig used on opensource projects

Example 6 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class JavaInstanceRunnable method setupInput.

private void setupInput(ContextImpl contextImpl) throws Exception {
    SourceSpec sourceSpec = this.instanceConfig.getFunctionDetails().getSource();
    Object object;
    // If source classname is not set, we default pulsar source
    if (sourceSpec.getClassName().isEmpty()) {
        Map<String, ConsumerConfig> topicSchema = new TreeMap<>();
        sourceSpec.getInputSpecsMap().forEach((topic, conf) -> {
            ConsumerConfig consumerConfig = ConsumerConfig.builder().isRegexPattern(conf.getIsRegexPattern()).build();
            if (conf.getSchemaType() != null && !conf.getSchemaType().isEmpty()) {
                consumerConfig.setSchemaType(conf.getSchemaType());
            } else if (conf.getSerdeClassName() != null && !conf.getSerdeClassName().isEmpty()) {
                consumerConfig.setSerdeClassName(conf.getSerdeClassName());
            }
            consumerConfig.setSchemaProperties(conf.getSchemaPropertiesMap());
            consumerConfig.setConsumerProperties(conf.getConsumerPropertiesMap());
            if (conf.hasReceiverQueueSize()) {
                consumerConfig.setReceiverQueueSize(conf.getReceiverQueueSize().getValue());
            }
            if (conf.hasCryptoSpec()) {
                consumerConfig.setCryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
            }
            consumerConfig.setPoolMessages(conf.getPoolMessages());
            topicSchema.put(topic, consumerConfig);
        });
        sourceSpec.getTopicsToSerDeClassNameMap().forEach((topic, serde) -> {
            topicSchema.put(topic, ConsumerConfig.builder().serdeClassName(serde).isRegexPattern(false).build());
        });
        if (!StringUtils.isEmpty(sourceSpec.getTopicsPattern())) {
            topicSchema.get(sourceSpec.getTopicsPattern()).setRegexPattern(true);
        }
        PulsarSourceConfig pulsarSourceConfig;
        // we can use a single consumer to read
        if (topicSchema.size() == 1) {
            SingleConsumerPulsarSourceConfig singleConsumerPulsarSourceConfig = new SingleConsumerPulsarSourceConfig();
            Map.Entry<String, ConsumerConfig> entry = topicSchema.entrySet().iterator().next();
            singleConsumerPulsarSourceConfig.setTopic(entry.getKey());
            singleConsumerPulsarSourceConfig.setConsumerConfig(entry.getValue());
            pulsarSourceConfig = singleConsumerPulsarSourceConfig;
        } else {
            MultiConsumerPulsarSourceConfig multiConsumerPulsarSourceConfig = new MultiConsumerPulsarSourceConfig();
            multiConsumerPulsarSourceConfig.setTopicSchema(topicSchema);
            pulsarSourceConfig = multiConsumerPulsarSourceConfig;
        }
        pulsarSourceConfig.setSubscriptionName(StringUtils.isNotBlank(sourceSpec.getSubscriptionName()) ? sourceSpec.getSubscriptionName() : InstanceUtils.getDefaultSubscriptionName(instanceConfig.getFunctionDetails()));
        pulsarSourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
        pulsarSourceConfig.setSubscriptionPosition(convertFromFunctionDetailsSubscriptionPosition(sourceSpec.getSubscriptionPosition()));
        checkNotNull(contextImpl.getSubscriptionType());
        pulsarSourceConfig.setSubscriptionType(contextImpl.getSubscriptionType());
        pulsarSourceConfig.setTypeClassName(sourceSpec.getTypeClassName());
        if (sourceSpec.getTimeoutMs() > 0) {
            pulsarSourceConfig.setTimeoutMs(sourceSpec.getTimeoutMs());
        }
        if (sourceSpec.getNegativeAckRedeliveryDelayMs() > 0) {
            pulsarSourceConfig.setNegativeAckRedeliveryDelayMs(sourceSpec.getNegativeAckRedeliveryDelayMs());
        }
        if (this.instanceConfig.getFunctionDetails().hasRetryDetails()) {
            pulsarSourceConfig.setMaxMessageRetries(this.instanceConfig.getFunctionDetails().getRetryDetails().getMaxMessageRetries());
            pulsarSourceConfig.setDeadLetterTopic(this.instanceConfig.getFunctionDetails().getRetryDetails().getDeadLetterTopic());
        }
        // that require messages to be put into an immediate queue
        if (pulsarSourceConfig instanceof SingleConsumerPulsarSourceConfig) {
            object = new SingleConsumerPulsarSource(this.client, (SingleConsumerPulsarSourceConfig) pulsarSourceConfig, this.properties, this.functionClassLoader);
        } else {
            object = new MultiConsumerPulsarSource(this.client, (MultiConsumerPulsarSourceConfig) pulsarSourceConfig, this.properties, this.functionClassLoader);
        }
    } else {
        // check if source is a batch source
        if (sourceSpec.getClassName().equals(BatchSourceExecutor.class.getName())) {
            object = Reflections.createInstance(sourceSpec.getClassName(), this.instanceClassLoader);
        } else {
            object = Reflections.createInstance(sourceSpec.getClassName(), this.functionClassLoader);
        }
    }
    Class<?>[] typeArgs;
    if (object instanceof Source) {
        typeArgs = TypeResolver.resolveRawArguments(Source.class, object.getClass());
        assert typeArgs.length > 0;
    } else {
        throw new RuntimeException("Source does not implement correct interface");
    }
    this.source = (Source<?>) object;
    if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) {
        Thread.currentThread().setContextClassLoader(this.functionClassLoader);
    }
    try {
        if (sourceSpec.getConfigs().isEmpty()) {
            this.source.open(new HashMap<>(), contextImpl);
        } else {
            this.source.open(ObjectMapperFactory.getThreadLocal().readValue(sourceSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
            }), contextImpl);
        }
        if (this.source instanceof PulsarSource) {
            contextImpl.setInputConsumers(((PulsarSource) this.source).getInputConsumers());
        }
    } catch (Exception e) {
        log.error("Source open produced uncaught exception: ", e);
        throw e;
    } finally {
        Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
    }
}

Example 7 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class PulsarSink method initializeSchema.

@SuppressWarnings("unchecked")
@VisibleForTesting
Schema<T> initializeSchema() throws ClassNotFoundException {
    if (StringUtils.isEmpty(this.pulsarSinkConfig.getTypeClassName())) {
        return (Schema<T>) Schema.BYTES;
    }
    Class<?> typeArg = Reflections.loadClass(this.pulsarSinkConfig.getTypeClassName(), functionClassLoader);
    if (Void.class.equals(typeArg)) {
        // return type is 'void', so there's no schema to check
        return null;
    }
    ConsumerConfig consumerConfig = new ConsumerConfig();
    consumerConfig.setSchemaProperties(pulsarSinkConfig.getSchemaProperties());
    if (!StringUtils.isEmpty(pulsarSinkConfig.getSchemaType())) {
        if (GenericRecord.class.isAssignableFrom(typeArg)) {
            consumerConfig.setSchemaType(SchemaType.AUTO_CONSUME.toString());
            SchemaType configuredSchemaType = SchemaType.valueOf(pulsarSinkConfig.getSchemaType());
            if (SchemaType.AUTO_CONSUME != configuredSchemaType) {
                log.info("The configured schema type {} is not able to write GenericRecords." + " So overwrite the schema type to be {}", configuredSchemaType, SchemaType.AUTO_CONSUME);
            }
        } else {
            consumerConfig.setSchemaType(pulsarSinkConfig.getSchemaType());
        }
        return (Schema<T>) topicSchema.getSchema(pulsarSinkConfig.getTopic(), typeArg, consumerConfig, false);
    } else {
        consumerConfig.setSchemaType(pulsarSinkConfig.getSerdeClassName());
        return (Schema<T>) topicSchema.getSchema(pulsarSinkConfig.getTopic(), typeArg, consumerConfig, false, functionClassLoader);
    }
}

Example 8 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class FunctionConfigUtils method convert.

public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) throws IllegalArgumentException {
    boolean isBuiltin = !org.apache.commons.lang3.StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN);
    Class<?>[] typeArgs = null;
    if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
        if (classLoader != null) {
            try {
                typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader);
            } catch (ClassNotFoundException | NoClassDefFoundError e) {
                throw new IllegalArgumentException(String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
            }
        }
    }
    FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
    // Setup source
    Function.SourceSpec.Builder sourceSpecBuilder = Function.SourceSpec.newBuilder();
    if (functionConfig.getInputs() != null) {
        functionConfig.getInputs().forEach((topicName -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setIsRegexPattern(false).build());
        }));
    }
    if (functionConfig.getTopicsPattern() != null && !functionConfig.getTopicsPattern().isEmpty()) {
        sourceSpecBuilder.putInputSpecs(functionConfig.getTopicsPattern(), Function.ConsumerSpec.newBuilder().setIsRegexPattern(true).build());
    }
    if (functionConfig.getCustomSerdeInputs() != null) {
        functionConfig.getCustomSerdeInputs().forEach((topicName, serdeClassName) -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSerdeClassName(serdeClassName).setIsRegexPattern(false).build());
        });
    }
    if (functionConfig.getCustomSchemaInputs() != null) {
        functionConfig.getCustomSchemaInputs().forEach((topicName, conf) -> {
            try {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSchemaType(consumerConfig.getSchemaType()).putAllSchemaProperties(consumerConfig.getSchemaProperties()).putAllConsumerProperties(consumerConfig.getConsumerProperties()).setIsRegexPattern(false).build());
            } catch (JsonProcessingException e) {
                throw new IllegalArgumentException(String.format("Incorrect custom schema inputs,Topic %s ", topicName));
            }
        });
    }
    if (functionConfig.getInputSpecs() != null) {
        functionConfig.getInputSpecs().forEach((topicName, consumerConf) -> {
            Function.ConsumerSpec.Builder bldr = Function.ConsumerSpec.newBuilder().setIsRegexPattern(consumerConf.isRegexPattern());
            if (isNotBlank(consumerConf.getSchemaType())) {
                bldr.setSchemaType(consumerConf.getSchemaType());
            } else if (isNotBlank(consumerConf.getSerdeClassName())) {
                bldr.setSerdeClassName(consumerConf.getSerdeClassName());
            }
            if (consumerConf.getReceiverQueueSize() != null) {
                bldr.setReceiverQueueSize(Function.ConsumerSpec.ReceiverQueueSize.newBuilder().setValue(consumerConf.getReceiverQueueSize()).build());
            }
            if (consumerConf.getSchemaProperties() != null) {
                bldr.putAllSchemaProperties(consumerConf.getSchemaProperties());
            }
            if (consumerConf.getCryptoConfig() != null) {
                bldr.setCryptoSpec(CryptoUtils.convert(consumerConf.getCryptoConfig()));
            }
            bldr.putAllConsumerProperties(consumerConf.getConsumerProperties());
            bldr.setPoolMessages(consumerConf.isPoolMessages());
            sourceSpecBuilder.putInputSpecs(topicName, bldr.build());
        });
    }
    // Set subscription type
    Function.SubscriptionType subType;
    if ((functionConfig.getRetainOrdering() != null && functionConfig.getRetainOrdering()) || FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE.equals(functionConfig.getProcessingGuarantees())) {
        subType = Function.SubscriptionType.FAILOVER;
    } else if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering()) {
        subType = Function.SubscriptionType.KEY_SHARED;
    } else {
        subType = Function.SubscriptionType.SHARED;
    }
    sourceSpecBuilder.setSubscriptionType(subType);
    // Set subscription name
    if (isNotBlank(functionConfig.getSubName())) {
        sourceSpecBuilder.setSubscriptionName(functionConfig.getSubName());
    }
    // Set subscription position
    if (functionConfig.getSubscriptionPosition() != null) {
        Function.SubscriptionPosition subPosition = null;
        if (SubscriptionInitialPosition.Earliest == functionConfig.getSubscriptionPosition()) {
            subPosition = Function.SubscriptionPosition.EARLIEST;
        } else {
            subPosition = Function.SubscriptionPosition.LATEST;
        }
        sourceSpecBuilder.setSubscriptionPosition(subPosition);
    }
    if (typeArgs != null) {
        sourceSpecBuilder.setTypeClassName(typeArgs[0].getName());
    }
    if (functionConfig.getTimeoutMs() != null) {
        sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs());
        // We use negative acks for fast tracking failures
        sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(functionConfig.getTimeoutMs());
    }
    if (functionConfig.getCleanupSubscription() != null) {
        sourceSpecBuilder.setCleanupSubscription(functionConfig.getCleanupSubscription());
    } else {
        sourceSpecBuilder.setCleanupSubscription(true);
    }
    functionDetailsBuilder.setSource(sourceSpecBuilder);
    // Setup sink
    Function.SinkSpec.Builder sinkSpecBuilder = Function.SinkSpec.newBuilder();
    if (functionConfig.getOutput() != null) {
        sinkSpecBuilder.setTopic(functionConfig.getOutput());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSerdeClassName())) {
        sinkSpecBuilder.setSerDeClassName(functionConfig.getOutputSerdeClassName());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSchemaType())) {
        sinkSpecBuilder.setSchemaType(functionConfig.getOutputSchemaType());
    }
    if (functionConfig.getForwardSourceMessageProperty() == Boolean.TRUE) {
        sinkSpecBuilder.setForwardSourceMessageProperty(functionConfig.getForwardSourceMessageProperty());
    }
    if (functionConfig.getCustomSchemaOutputs() != null && functionConfig.getOutput() != null) {
        String conf = functionConfig.getCustomSchemaOutputs().get(functionConfig.getOutput());
        try {
            if (StringUtils.isNotEmpty(conf)) {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sinkSpecBuilder.putAllSchemaProperties(consumerConfig.getSchemaProperties());
                sinkSpecBuilder.putAllConsumerProperties(consumerConfig.getConsumerProperties());
            }
        } catch (JsonProcessingException e) {
            throw new IllegalArgumentException(String.format("Incorrect custom schema outputs,Topic %s ", functionConfig.getOutput()));
        }
    }
    if (typeArgs != null) {
        sinkSpecBuilder.setTypeClassName(typeArgs[1].getName());
    }
    if (functionConfig.getProducerConfig() != null) {
        ProducerConfig producerConf = functionConfig.getProducerConfig();
        Function.ProducerSpec.Builder pbldr = Function.ProducerSpec.newBuilder();
        if (producerConf.getMaxPendingMessages() != null) {
            pbldr.setMaxPendingMessages(producerConf.getMaxPendingMessages());
        }
        if (producerConf.getMaxPendingMessagesAcrossPartitions() != null) {
            pbldr.setMaxPendingMessagesAcrossPartitions(producerConf.getMaxPendingMessagesAcrossPartitions());
        }
        if (producerConf.getUseThreadLocalProducers() != null) {
            pbldr.setUseThreadLocalProducers(producerConf.getUseThreadLocalProducers());
        }
        if (producerConf.getCryptoConfig() != null) {
            pbldr.setCryptoSpec(CryptoUtils.convert(producerConf.getCryptoConfig()));
        }
        if (producerConf.getBatchBuilder() != null) {
            pbldr.setBatchBuilder(producerConf.getBatchBuilder());
        }
        sinkSpecBuilder.setProducerSpec(pbldr.build());
    }
    functionDetailsBuilder.setSink(sinkSpecBuilder);
    if (functionConfig.getTenant() != null) {
        functionDetailsBuilder.setTenant(functionConfig.getTenant());
    }
    if (functionConfig.getNamespace() != null) {
        functionDetailsBuilder.setNamespace(functionConfig.getNamespace());
    }
    if (functionConfig.getName() != null) {
        functionDetailsBuilder.setName(functionConfig.getName());
    }
    if (functionConfig.getLogTopic() != null) {
        functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic());
    }
    if (functionConfig.getRuntime() != null) {
        functionDetailsBuilder.setRuntime(FunctionCommon.convertRuntime(functionConfig.getRuntime()));
    }
    if (functionConfig.getProcessingGuarantees() != null) {
        functionDetailsBuilder.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionConfig.getProcessingGuarantees()));
    }
    if (functionConfig.getRetainKeyOrdering() != null) {
        functionDetailsBuilder.setRetainKeyOrdering(functionConfig.getRetainKeyOrdering());
    }
    if (functionConfig.getRetainOrdering() != null) {
        functionDetailsBuilder.setRetainOrdering(functionConfig.getRetainOrdering());
    }
    if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0) {
        Function.RetryDetails.Builder retryBuilder = Function.RetryDetails.newBuilder();
        retryBuilder.setMaxMessageRetries(functionConfig.getMaxMessageRetries());
        if (isNotEmpty(functionConfig.getDeadLetterTopic())) {
            retryBuilder.setDeadLetterTopic(functionConfig.getDeadLetterTopic());
        }
        functionDetailsBuilder.setRetryDetails(retryBuilder);
    }
    Map<String, Object> configs = new HashMap<>();
    if (functionConfig.getUserConfig() != null) {
        configs.putAll(functionConfig.getUserConfig());
    }
    // windowing related
    WindowConfig windowConfig = functionConfig.getWindowConfig();
    if (windowConfig != null) {
        windowConfig.setActualWindowFunctionClassName(functionConfig.getClassName());
        configs.put(WindowConfig.WINDOW_CONFIG_KEY, windowConfig);
        // set class name to window function executor
        functionDetailsBuilder.setClassName("org.apache.pulsar.functions.windowing.WindowFunctionExecutor");
    } else {
        if (functionConfig.getClassName() != null) {
            functionDetailsBuilder.setClassName(functionConfig.getClassName());
        }
    }
    if (!configs.isEmpty()) {
        functionDetailsBuilder.setUserConfig(new Gson().toJson(configs));
    }
    if (functionConfig.getSecrets() != null && !functionConfig.getSecrets().isEmpty()) {
        functionDetailsBuilder.setSecretsMap(new Gson().toJson(functionConfig.getSecrets()));
    }
    if (functionConfig.getAutoAck() != null) {
        functionDetailsBuilder.setAutoAck(functionConfig.getAutoAck());
    } else {
        functionDetailsBuilder.setAutoAck(true);
    }
    if (functionConfig.getParallelism() != null) {
        functionDetailsBuilder.setParallelism(functionConfig.getParallelism());
    } else {
        functionDetailsBuilder.setParallelism(1);
    }
    // use default resources if resources not set
    Resources resources = Resources.mergeWithDefault(functionConfig.getResources());
    Function.Resources.Builder bldr = Function.Resources.newBuilder();
    bldr.setCpu(resources.getCpu());
    bldr.setRam(resources.getRam());
    bldr.setDisk(resources.getDisk());
    functionDetailsBuilder.setResources(bldr);
    if (!StringUtils.isEmpty(functionConfig.getRuntimeFlags())) {
        functionDetailsBuilder.setRuntimeFlags(functionConfig.getRuntimeFlags());
    }
    functionDetailsBuilder.setComponentType(FunctionDetails.ComponentType.FUNCTION);
    if (!StringUtils.isEmpty(functionConfig.getCustomRuntimeOptions())) {
        functionDetailsBuilder.setCustomRuntimeOptions(functionConfig.getCustomRuntimeOptions());
    }
    if (isBuiltin) {
        String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
        functionDetailsBuilder.setBuiltin(builtin);
    }
    return functionDetailsBuilder.build();
}

Example 9 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class SinkConfigUtils method convertFromDetails.

public static SinkConfig convertFromDetails(FunctionDetails functionDetails) {
    SinkConfig sinkConfig = new SinkConfig();
    sinkConfig.setTenant(functionDetails.getTenant());
    sinkConfig.setNamespace(functionDetails.getNamespace());
    sinkConfig.setName(functionDetails.getName());
    sinkConfig.setParallelism(functionDetails.getParallelism());
    sinkConfig.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionDetails.getProcessingGuarantees()));
    Map<String, ConsumerConfig> consumerConfigMap = new HashMap<>();
    List<String> inputs = new ArrayList<>();
    for (Map.Entry<String, Function.ConsumerSpec> input : functionDetails.getSource().getInputSpecsMap().entrySet()) {
        ConsumerConfig consumerConfig = new ConsumerConfig();
        if (!isEmpty(input.getValue().getSerdeClassName())) {
            consumerConfig.setSerdeClassName(input.getValue().getSerdeClassName());
        }
        if (!isEmpty(input.getValue().getSchemaType())) {
            consumerConfig.setSchemaType(input.getValue().getSchemaType());
        }
        if (input.getValue().hasReceiverQueueSize()) {
            consumerConfig.setReceiverQueueSize(input.getValue().getReceiverQueueSize().getValue());
        }
        if (input.getValue().hasCryptoSpec()) {
            consumerConfig.setCryptoConfig(CryptoUtils.convertFromSpec(input.getValue().getCryptoSpec()));
        }
        consumerConfig.setRegexPattern(input.getValue().getIsRegexPattern());
        consumerConfig.setConsumerProperties(input.getValue().getConsumerPropertiesMap());
        consumerConfig.setPoolMessages(input.getValue().getPoolMessages());
        consumerConfigMap.put(input.getKey(), consumerConfig);
        inputs.add(input.getKey());
    }
    sinkConfig.setInputs(inputs);
    sinkConfig.setInputSpecs(consumerConfigMap);
    if (!isEmpty(functionDetails.getSource().getSubscriptionName())) {
        sinkConfig.setSourceSubscriptionName(functionDetails.getSource().getSubscriptionName());
    }
    if (functionDetails.getSource().getSubscriptionType() == Function.SubscriptionType.FAILOVER) {
        sinkConfig.setRetainOrdering(true);
        sinkConfig.setRetainKeyOrdering(false);
    } else if (functionDetails.getSource().getSubscriptionType() == Function.SubscriptionType.KEY_SHARED) {
        sinkConfig.setRetainOrdering(false);
        sinkConfig.setRetainKeyOrdering(true);
    } else {
        sinkConfig.setRetainOrdering(false);
        sinkConfig.setRetainKeyOrdering(false);
    }
    sinkConfig.setProcessingGuarantees(convertProcessingGuarantee(functionDetails.getProcessingGuarantees()));
    sinkConfig.setAutoAck(functionDetails.getAutoAck());
    sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription());
    // Set subscription position
    sinkConfig.setSourceSubscriptionPosition(convertFromFunctionDetailsSubscriptionPosition(functionDetails.getSource().getSubscriptionPosition()));
    sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription());
    if (functionDetails.getSource().getTimeoutMs() != 0) {
        sinkConfig.setTimeoutMs(functionDetails.getSource().getTimeoutMs());
    }
    if (functionDetails.getSource().getNegativeAckRedeliveryDelayMs() > 0) {
        sinkConfig.setNegativeAckRedeliveryDelayMs(functionDetails.getSource().getNegativeAckRedeliveryDelayMs());
    }
    if (!isEmpty(functionDetails.getSink().getClassName())) {
        sinkConfig.setClassName(functionDetails.getSink().getClassName());
    }
    if (!isEmpty(functionDetails.getSink().getBuiltin())) {
        sinkConfig.setArchive("builtin://" + functionDetails.getSink().getBuiltin());
    }
    if (!org.apache.commons.lang3.StringUtils.isEmpty(functionDetails.getSink().getConfigs())) {
        TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {
        };
        Map<String, Object> configMap;
        try {
            configMap = ObjectMapperFactory.getThreadLocal().readValue(functionDetails.getSink().getConfigs(), typeRef);
        } catch (IOException e) {
            log.error("Failed to read configs for sink {}", FunctionCommon.getFullyQualifiedName(functionDetails), e);
            throw new RuntimeException(e);
        }
        sinkConfig.setConfigs(configMap);
    }
    if (!isEmpty(functionDetails.getSecretsMap())) {
        Type type = new TypeToken<Map<String, Object>>() {
        }.getType();
        Map<String, Object> secretsMap = new Gson().fromJson(functionDetails.getSecretsMap(), type);
        sinkConfig.setSecrets(secretsMap);
    }
    if (functionDetails.hasResources()) {
        Resources resources = new Resources();
        resources.setCpu(functionDetails.getResources().getCpu());
        resources.setRam(functionDetails.getResources().getRam());
        resources.setDisk(functionDetails.getResources().getDisk());
        sinkConfig.setResources(resources);
    }
    if (isNotBlank(functionDetails.getRuntimeFlags())) {
        sinkConfig.setRuntimeFlags(functionDetails.getRuntimeFlags());
    }
    if (!isEmpty(functionDetails.getCustomRuntimeOptions())) {
        sinkConfig.setCustomRuntimeOptions(functionDetails.getCustomRuntimeOptions());
    }
    if (functionDetails.hasRetryDetails()) {
        sinkConfig.setMaxMessageRetries(functionDetails.getRetryDetails().getMaxMessageRetries());
        if (StringUtils.isNotBlank(functionDetails.getRetryDetails().getDeadLetterTopic())) {
            sinkConfig.setDeadLetterTopic(functionDetails.getRetryDetails().getDeadLetterTopic());
        }
    }
    return sinkConfig;
}

Example 10 with ConsumerConfig

use of org.apache.pulsar.common.functions.ConsumerConfig in project pulsar by apache.

the class PulsarSourceTest method testInconsistentInputType.

/**
 * Verify that function input type should be consistent with input serde type.
 */
@Test(dataProvider = "sourceImpls")
public void testInconsistentInputType(PulsarSourceConfig pulsarSourceConfig) throws Exception {
    // set type to be inconsistent to that of SerDe
    pulsarSourceConfig.setTypeClassName(Integer.class.getName());
    String topic = "persistent://sample/ns1/test_result";
    ConsumerConfig consumerConfig = ConsumerConfig.builder().serdeClassName(TestSerDe.class.getName()).build();
    setTopicAndConsumerConfig(pulsarSourceConfig, topic, consumerConfig);
    @Cleanup PulsarSource<?> pulsarSource = getPulsarSource(pulsarSourceConfig);
    try {
        pulsarSource.open(new HashMap<>(), Mockito.mock(SourceContext.class));
        fail("Should fail constructing java instance if function type is inconsistent with serde type");
    } catch (RuntimeException ex) {
        log.error("RuntimeException: {}", ex, ex);
        assertTrue(ex.getMessage().startsWith("Inconsistent types found between function input type and serde type:"));
    } catch (Exception ex) {
        log.error("Exception: {}", ex, ex);
        fail();
    }
}