Search in sources :

Example 1 with BUILTIN

use of org.apache.pulsar.common.functions.Utils.BUILTIN in project pulsar by apache.

the class FunctionConfigUtils method convert.

public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) throws IllegalArgumentException {
    boolean isBuiltin = !org.apache.commons.lang3.StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN);
    Class<?>[] typeArgs = null;
    if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
        if (classLoader != null) {
            try {
                typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader);
            } catch (ClassNotFoundException | NoClassDefFoundError e) {
                throw new IllegalArgumentException(String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
            }
        }
    }
    FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
    // Setup source
    Function.SourceSpec.Builder sourceSpecBuilder = Function.SourceSpec.newBuilder();
    if (functionConfig.getInputs() != null) {
        functionConfig.getInputs().forEach((topicName -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setIsRegexPattern(false).build());
        }));
    }
    if (functionConfig.getTopicsPattern() != null && !functionConfig.getTopicsPattern().isEmpty()) {
        sourceSpecBuilder.putInputSpecs(functionConfig.getTopicsPattern(), Function.ConsumerSpec.newBuilder().setIsRegexPattern(true).build());
    }
    if (functionConfig.getCustomSerdeInputs() != null) {
        functionConfig.getCustomSerdeInputs().forEach((topicName, serdeClassName) -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSerdeClassName(serdeClassName).setIsRegexPattern(false).build());
        });
    }
    if (functionConfig.getCustomSchemaInputs() != null) {
        functionConfig.getCustomSchemaInputs().forEach((topicName, conf) -> {
            try {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSchemaType(consumerConfig.getSchemaType()).putAllSchemaProperties(consumerConfig.getSchemaProperties()).putAllConsumerProperties(consumerConfig.getConsumerProperties()).setIsRegexPattern(false).build());
            } catch (JsonProcessingException e) {
                throw new IllegalArgumentException(String.format("Incorrect custom schema inputs,Topic %s ", topicName));
            }
        });
    }
    if (functionConfig.getInputSpecs() != null) {
        functionConfig.getInputSpecs().forEach((topicName, consumerConf) -> {
            Function.ConsumerSpec.Builder bldr = Function.ConsumerSpec.newBuilder().setIsRegexPattern(consumerConf.isRegexPattern());
            if (isNotBlank(consumerConf.getSchemaType())) {
                bldr.setSchemaType(consumerConf.getSchemaType());
            } else if (isNotBlank(consumerConf.getSerdeClassName())) {
                bldr.setSerdeClassName(consumerConf.getSerdeClassName());
            }
            if (consumerConf.getReceiverQueueSize() != null) {
                bldr.setReceiverQueueSize(Function.ConsumerSpec.ReceiverQueueSize.newBuilder().setValue(consumerConf.getReceiverQueueSize()).build());
            }
            if (consumerConf.getSchemaProperties() != null) {
                bldr.putAllSchemaProperties(consumerConf.getSchemaProperties());
            }
            if (consumerConf.getCryptoConfig() != null) {
                bldr.setCryptoSpec(CryptoUtils.convert(consumerConf.getCryptoConfig()));
            }
            bldr.putAllConsumerProperties(consumerConf.getConsumerProperties());
            bldr.setPoolMessages(consumerConf.isPoolMessages());
            sourceSpecBuilder.putInputSpecs(topicName, bldr.build());
        });
    }
    // Set subscription type
    Function.SubscriptionType subType;
    if ((functionConfig.getRetainOrdering() != null && functionConfig.getRetainOrdering()) || FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE.equals(functionConfig.getProcessingGuarantees())) {
        subType = Function.SubscriptionType.FAILOVER;
    } else if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering()) {
        subType = Function.SubscriptionType.KEY_SHARED;
    } else {
        subType = Function.SubscriptionType.SHARED;
    }
    sourceSpecBuilder.setSubscriptionType(subType);
    // Set subscription name
    if (isNotBlank(functionConfig.getSubName())) {
        sourceSpecBuilder.setSubscriptionName(functionConfig.getSubName());
    }
    // Set subscription position
    if (functionConfig.getSubscriptionPosition() != null) {
        Function.SubscriptionPosition subPosition = null;
        if (SubscriptionInitialPosition.Earliest == functionConfig.getSubscriptionPosition()) {
            subPosition = Function.SubscriptionPosition.EARLIEST;
        } else {
            subPosition = Function.SubscriptionPosition.LATEST;
        }
        sourceSpecBuilder.setSubscriptionPosition(subPosition);
    }
    if (typeArgs != null) {
        sourceSpecBuilder.setTypeClassName(typeArgs[0].getName());
    }
    if (functionConfig.getTimeoutMs() != null) {
        sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs());
        // We use negative acks for fast tracking failures
        sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(functionConfig.getTimeoutMs());
    }
    if (functionConfig.getCleanupSubscription() != null) {
        sourceSpecBuilder.setCleanupSubscription(functionConfig.getCleanupSubscription());
    } else {
        sourceSpecBuilder.setCleanupSubscription(true);
    }
    functionDetailsBuilder.setSource(sourceSpecBuilder);
    // Setup sink
    Function.SinkSpec.Builder sinkSpecBuilder = Function.SinkSpec.newBuilder();
    if (functionConfig.getOutput() != null) {
        sinkSpecBuilder.setTopic(functionConfig.getOutput());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSerdeClassName())) {
        sinkSpecBuilder.setSerDeClassName(functionConfig.getOutputSerdeClassName());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSchemaType())) {
        sinkSpecBuilder.setSchemaType(functionConfig.getOutputSchemaType());
    }
    if (functionConfig.getForwardSourceMessageProperty() == Boolean.TRUE) {
        sinkSpecBuilder.setForwardSourceMessageProperty(functionConfig.getForwardSourceMessageProperty());
    }
    if (functionConfig.getCustomSchemaOutputs() != null && functionConfig.getOutput() != null) {
        String conf = functionConfig.getCustomSchemaOutputs().get(functionConfig.getOutput());
        try {
            if (StringUtils.isNotEmpty(conf)) {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sinkSpecBuilder.putAllSchemaProperties(consumerConfig.getSchemaProperties());
                sinkSpecBuilder.putAllConsumerProperties(consumerConfig.getConsumerProperties());
            }
        } catch (JsonProcessingException e) {
            throw new IllegalArgumentException(String.format("Incorrect custom schema outputs,Topic %s ", functionConfig.getOutput()));
        }
    }
    if (typeArgs != null) {
        sinkSpecBuilder.setTypeClassName(typeArgs[1].getName());
    }
    if (functionConfig.getProducerConfig() != null) {
        ProducerConfig producerConf = functionConfig.getProducerConfig();
        Function.ProducerSpec.Builder pbldr = Function.ProducerSpec.newBuilder();
        if (producerConf.getMaxPendingMessages() != null) {
            pbldr.setMaxPendingMessages(producerConf.getMaxPendingMessages());
        }
        if (producerConf.getMaxPendingMessagesAcrossPartitions() != null) {
            pbldr.setMaxPendingMessagesAcrossPartitions(producerConf.getMaxPendingMessagesAcrossPartitions());
        }
        if (producerConf.getUseThreadLocalProducers() != null) {
            pbldr.setUseThreadLocalProducers(producerConf.getUseThreadLocalProducers());
        }
        if (producerConf.getCryptoConfig() != null) {
            pbldr.setCryptoSpec(CryptoUtils.convert(producerConf.getCryptoConfig()));
        }
        if (producerConf.getBatchBuilder() != null) {
            pbldr.setBatchBuilder(producerConf.getBatchBuilder());
        }
        sinkSpecBuilder.setProducerSpec(pbldr.build());
    }
    functionDetailsBuilder.setSink(sinkSpecBuilder);
    if (functionConfig.getTenant() != null) {
        functionDetailsBuilder.setTenant(functionConfig.getTenant());
    }
    if (functionConfig.getNamespace() != null) {
        functionDetailsBuilder.setNamespace(functionConfig.getNamespace());
    }
    if (functionConfig.getName() != null) {
        functionDetailsBuilder.setName(functionConfig.getName());
    }
    if (functionConfig.getLogTopic() != null) {
        functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic());
    }
    if (functionConfig.getRuntime() != null) {
        functionDetailsBuilder.setRuntime(FunctionCommon.convertRuntime(functionConfig.getRuntime()));
    }
    if (functionConfig.getProcessingGuarantees() != null) {
        functionDetailsBuilder.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionConfig.getProcessingGuarantees()));
    }
    if (functionConfig.getRetainKeyOrdering() != null) {
        functionDetailsBuilder.setRetainKeyOrdering(functionConfig.getRetainKeyOrdering());
    }
    if (functionConfig.getRetainOrdering() != null) {
        functionDetailsBuilder.setRetainOrdering(functionConfig.getRetainOrdering());
    }
    if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0) {
        Function.RetryDetails.Builder retryBuilder = Function.RetryDetails.newBuilder();
        retryBuilder.setMaxMessageRetries(functionConfig.getMaxMessageRetries());
        if (isNotEmpty(functionConfig.getDeadLetterTopic())) {
            retryBuilder.setDeadLetterTopic(functionConfig.getDeadLetterTopic());
        }
        functionDetailsBuilder.setRetryDetails(retryBuilder);
    }
    Map<String, Object> configs = new HashMap<>();
    if (functionConfig.getUserConfig() != null) {
        configs.putAll(functionConfig.getUserConfig());
    }
    // windowing related
    WindowConfig windowConfig = functionConfig.getWindowConfig();
    if (windowConfig != null) {
        windowConfig.setActualWindowFunctionClassName(functionConfig.getClassName());
        configs.put(WindowConfig.WINDOW_CONFIG_KEY, windowConfig);
        // set class name to window function executor
        functionDetailsBuilder.setClassName("org.apache.pulsar.functions.windowing.WindowFunctionExecutor");
    } else {
        if (functionConfig.getClassName() != null) {
            functionDetailsBuilder.setClassName(functionConfig.getClassName());
        }
    }
    if (!configs.isEmpty()) {
        functionDetailsBuilder.setUserConfig(new Gson().toJson(configs));
    }
    if (functionConfig.getSecrets() != null && !functionConfig.getSecrets().isEmpty()) {
        functionDetailsBuilder.setSecretsMap(new Gson().toJson(functionConfig.getSecrets()));
    }
    if (functionConfig.getAutoAck() != null) {
        functionDetailsBuilder.setAutoAck(functionConfig.getAutoAck());
    } else {
        functionDetailsBuilder.setAutoAck(true);
    }
    if (functionConfig.getParallelism() != null) {
        functionDetailsBuilder.setParallelism(functionConfig.getParallelism());
    } else {
        functionDetailsBuilder.setParallelism(1);
    }
    // use default resources if resources not set
    Resources resources = Resources.mergeWithDefault(functionConfig.getResources());
    Function.Resources.Builder bldr = Function.Resources.newBuilder();
    bldr.setCpu(resources.getCpu());
    bldr.setRam(resources.getRam());
    bldr.setDisk(resources.getDisk());
    functionDetailsBuilder.setResources(bldr);
    if (!StringUtils.isEmpty(functionConfig.getRuntimeFlags())) {
        functionDetailsBuilder.setRuntimeFlags(functionConfig.getRuntimeFlags());
    }
    functionDetailsBuilder.setComponentType(FunctionDetails.ComponentType.FUNCTION);
    if (!StringUtils.isEmpty(functionConfig.getCustomRuntimeOptions())) {
        functionDetailsBuilder.setCustomRuntimeOptions(functionConfig.getCustomRuntimeOptions());
    }
    if (isBuiltin) {
        String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
        functionDetailsBuilder.setBuiltin(builtin);
    }
    return functionDetailsBuilder.build();
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) TypeToken(com.google.gson.reflect.TypeToken) TopicName(org.apache.pulsar.common.naming.TopicName) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) StringUtils.isNotEmpty(org.apache.commons.lang.StringUtils.isNotEmpty) HashMap(java.util.HashMap) Resources(org.apache.pulsar.common.functions.Resources) ClassLoaderUtils.loadJar(org.apache.pulsar.common.util.ClassLoaderUtils.loadJar) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) StringUtils.isBlank(org.apache.commons.lang.StringUtils.isBlank) FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails) Gson(com.google.gson.Gson) Map(java.util.Map) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) LinkedList(java.util.LinkedList) StringUtils.isEmpty(org.apache.commons.lang3.StringUtils.isEmpty) FunctionCommon.convertFromFunctionDetailsSubscriptionPosition(org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition) Function(org.apache.pulsar.functions.proto.Function) WindowConfig(org.apache.pulsar.common.functions.WindowConfig) BUILTIN(org.apache.pulsar.common.functions.Utils.BUILTIN) MalformedURLException(java.net.MalformedURLException) Collection(java.util.Collection) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) StringUtils.isNotBlank(org.apache.commons.lang.StringUtils.isNotBlank) File(java.io.File) ProducerConfig(org.apache.pulsar.common.functions.ProducerConfig) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Type(java.lang.reflect.Type) HashMap(java.util.HashMap) Gson(com.google.gson.Gson) Function(org.apache.pulsar.functions.proto.Function) ProducerConfig(org.apache.pulsar.common.functions.ProducerConfig) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) WindowConfig(org.apache.pulsar.common.functions.WindowConfig) FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails) Resources(org.apache.pulsar.common.functions.Resources)

Example 2 with BUILTIN

use of org.apache.pulsar.common.functions.Utils.BUILTIN in project incubator-pulsar by apache.

the class FunctionConfigUtils method convert.

public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) throws IllegalArgumentException {
    boolean isBuiltin = !org.apache.commons.lang3.StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN);
    Class<?>[] typeArgs = null;
    if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
        if (classLoader != null) {
            try {
                typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader);
            } catch (ClassNotFoundException | NoClassDefFoundError e) {
                throw new IllegalArgumentException(String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
            }
        }
    }
    FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
    // Setup source
    Function.SourceSpec.Builder sourceSpecBuilder = Function.SourceSpec.newBuilder();
    if (functionConfig.getInputs() != null) {
        functionConfig.getInputs().forEach((topicName -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setIsRegexPattern(false).build());
        }));
    }
    if (functionConfig.getTopicsPattern() != null && !functionConfig.getTopicsPattern().isEmpty()) {
        sourceSpecBuilder.putInputSpecs(functionConfig.getTopicsPattern(), Function.ConsumerSpec.newBuilder().setIsRegexPattern(true).build());
    }
    if (functionConfig.getCustomSerdeInputs() != null) {
        functionConfig.getCustomSerdeInputs().forEach((topicName, serdeClassName) -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSerdeClassName(serdeClassName).setIsRegexPattern(false).build());
        });
    }
    if (functionConfig.getCustomSchemaInputs() != null) {
        functionConfig.getCustomSchemaInputs().forEach((topicName, conf) -> {
            try {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSchemaType(consumerConfig.getSchemaType()).putAllSchemaProperties(consumerConfig.getSchemaProperties()).putAllConsumerProperties(consumerConfig.getConsumerProperties()).setIsRegexPattern(false).build());
            } catch (JsonProcessingException e) {
                throw new IllegalArgumentException(String.format("Incorrect custom schema inputs,Topic %s ", topicName));
            }
        });
    }
    if (functionConfig.getInputSpecs() != null) {
        functionConfig.getInputSpecs().forEach((topicName, consumerConf) -> {
            Function.ConsumerSpec.Builder bldr = Function.ConsumerSpec.newBuilder().setIsRegexPattern(consumerConf.isRegexPattern());
            if (isNotBlank(consumerConf.getSchemaType())) {
                bldr.setSchemaType(consumerConf.getSchemaType());
            } else if (isNotBlank(consumerConf.getSerdeClassName())) {
                bldr.setSerdeClassName(consumerConf.getSerdeClassName());
            }
            if (consumerConf.getReceiverQueueSize() != null) {
                bldr.setReceiverQueueSize(Function.ConsumerSpec.ReceiverQueueSize.newBuilder().setValue(consumerConf.getReceiverQueueSize()).build());
            }
            if (consumerConf.getSchemaProperties() != null) {
                bldr.putAllSchemaProperties(consumerConf.getSchemaProperties());
            }
            if (consumerConf.getCryptoConfig() != null) {
                bldr.setCryptoSpec(CryptoUtils.convert(consumerConf.getCryptoConfig()));
            }
            bldr.putAllConsumerProperties(consumerConf.getConsumerProperties());
            bldr.setPoolMessages(consumerConf.isPoolMessages());
            sourceSpecBuilder.putInputSpecs(topicName, bldr.build());
        });
    }
    // Set subscription type
    Function.SubscriptionType subType;
    if ((functionConfig.getRetainOrdering() != null && functionConfig.getRetainOrdering()) || FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE.equals(functionConfig.getProcessingGuarantees())) {
        subType = Function.SubscriptionType.FAILOVER;
    } else if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering()) {
        subType = Function.SubscriptionType.KEY_SHARED;
    } else {
        subType = Function.SubscriptionType.SHARED;
    }
    sourceSpecBuilder.setSubscriptionType(subType);
    // Set subscription name
    if (isNotBlank(functionConfig.getSubName())) {
        sourceSpecBuilder.setSubscriptionName(functionConfig.getSubName());
    }
    // Set subscription position
    if (functionConfig.getSubscriptionPosition() != null) {
        Function.SubscriptionPosition subPosition = null;
        if (SubscriptionInitialPosition.Earliest == functionConfig.getSubscriptionPosition()) {
            subPosition = Function.SubscriptionPosition.EARLIEST;
        } else {
            subPosition = Function.SubscriptionPosition.LATEST;
        }
        sourceSpecBuilder.setSubscriptionPosition(subPosition);
    }
    if (typeArgs != null) {
        sourceSpecBuilder.setTypeClassName(typeArgs[0].getName());
    }
    if (functionConfig.getTimeoutMs() != null) {
        sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs());
        // We use negative acks for fast tracking failures
        sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(functionConfig.getTimeoutMs());
    }
    if (functionConfig.getCleanupSubscription() != null) {
        sourceSpecBuilder.setCleanupSubscription(functionConfig.getCleanupSubscription());
    } else {
        sourceSpecBuilder.setCleanupSubscription(true);
    }
    functionDetailsBuilder.setSource(sourceSpecBuilder);
    // Setup sink
    Function.SinkSpec.Builder sinkSpecBuilder = Function.SinkSpec.newBuilder();
    if (functionConfig.getOutput() != null) {
        sinkSpecBuilder.setTopic(functionConfig.getOutput());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSerdeClassName())) {
        sinkSpecBuilder.setSerDeClassName(functionConfig.getOutputSerdeClassName());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSchemaType())) {
        sinkSpecBuilder.setSchemaType(functionConfig.getOutputSchemaType());
    }
    if (functionConfig.getForwardSourceMessageProperty() == Boolean.TRUE) {
        sinkSpecBuilder.setForwardSourceMessageProperty(functionConfig.getForwardSourceMessageProperty());
    }
    if (functionConfig.getCustomSchemaOutputs() != null && functionConfig.getOutput() != null) {
        String conf = functionConfig.getCustomSchemaOutputs().get(functionConfig.getOutput());
        try {
            if (StringUtils.isNotEmpty(conf)) {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sinkSpecBuilder.putAllSchemaProperties(consumerConfig.getSchemaProperties());
                sinkSpecBuilder.putAllConsumerProperties(consumerConfig.getConsumerProperties());
            }
        } catch (JsonProcessingException e) {
            throw new IllegalArgumentException(String.format("Incorrect custom schema outputs,Topic %s ", functionConfig.getOutput()));
        }
    }
    if (typeArgs != null) {
        sinkSpecBuilder.setTypeClassName(typeArgs[1].getName());
    }
    if (functionConfig.getProducerConfig() != null) {
        ProducerConfig producerConf = functionConfig.getProducerConfig();
        Function.ProducerSpec.Builder pbldr = Function.ProducerSpec.newBuilder();
        if (producerConf.getMaxPendingMessages() != null) {
            pbldr.setMaxPendingMessages(producerConf.getMaxPendingMessages());
        }
        if (producerConf.getMaxPendingMessagesAcrossPartitions() != null) {
            pbldr.setMaxPendingMessagesAcrossPartitions(producerConf.getMaxPendingMessagesAcrossPartitions());
        }
        if (producerConf.getUseThreadLocalProducers() != null) {
            pbldr.setUseThreadLocalProducers(producerConf.getUseThreadLocalProducers());
        }
        if (producerConf.getCryptoConfig() != null) {
            pbldr.setCryptoSpec(CryptoUtils.convert(producerConf.getCryptoConfig()));
        }
        if (producerConf.getBatchBuilder() != null) {
            pbldr.setBatchBuilder(producerConf.getBatchBuilder());
        }
        sinkSpecBuilder.setProducerSpec(pbldr.build());
    }
    functionDetailsBuilder.setSink(sinkSpecBuilder);
    if (functionConfig.getTenant() != null) {
        functionDetailsBuilder.setTenant(functionConfig.getTenant());
    }
    if (functionConfig.getNamespace() != null) {
        functionDetailsBuilder.setNamespace(functionConfig.getNamespace());
    }
    if (functionConfig.getName() != null) {
        functionDetailsBuilder.setName(functionConfig.getName());
    }
    if (functionConfig.getLogTopic() != null) {
        functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic());
    }
    if (functionConfig.getRuntime() != null) {
        functionDetailsBuilder.setRuntime(FunctionCommon.convertRuntime(functionConfig.getRuntime()));
    }
    if (functionConfig.getProcessingGuarantees() != null) {
        functionDetailsBuilder.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionConfig.getProcessingGuarantees()));
    }
    if (functionConfig.getRetainKeyOrdering() != null) {
        functionDetailsBuilder.setRetainKeyOrdering(functionConfig.getRetainKeyOrdering());
    }
    if (functionConfig.getRetainOrdering() != null) {
        functionDetailsBuilder.setRetainOrdering(functionConfig.getRetainOrdering());
    }
    if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0) {
        Function.RetryDetails.Builder retryBuilder = Function.RetryDetails.newBuilder();
        retryBuilder.setMaxMessageRetries(functionConfig.getMaxMessageRetries());
        if (isNotEmpty(functionConfig.getDeadLetterTopic())) {
            retryBuilder.setDeadLetterTopic(functionConfig.getDeadLetterTopic());
        }
        functionDetailsBuilder.setRetryDetails(retryBuilder);
    }
    Map<String, Object> configs = new HashMap<>();
    if (functionConfig.getUserConfig() != null) {
        configs.putAll(functionConfig.getUserConfig());
    }
    // windowing related
    WindowConfig windowConfig = functionConfig.getWindowConfig();
    if (windowConfig != null) {
        windowConfig.setActualWindowFunctionClassName(functionConfig.getClassName());
        configs.put(WindowConfig.WINDOW_CONFIG_KEY, windowConfig);
        // set class name to window function executor
        functionDetailsBuilder.setClassName("org.apache.pulsar.functions.windowing.WindowFunctionExecutor");
    } else {
        if (functionConfig.getClassName() != null) {
            functionDetailsBuilder.setClassName(functionConfig.getClassName());
        }
    }
    if (!configs.isEmpty()) {
        functionDetailsBuilder.setUserConfig(new Gson().toJson(configs));
    }
    if (functionConfig.getSecrets() != null && !functionConfig.getSecrets().isEmpty()) {
        functionDetailsBuilder.setSecretsMap(new Gson().toJson(functionConfig.getSecrets()));
    }
    if (functionConfig.getAutoAck() != null) {
        functionDetailsBuilder.setAutoAck(functionConfig.getAutoAck());
    } else {
        functionDetailsBuilder.setAutoAck(true);
    }
    if (functionConfig.getParallelism() != null) {
        functionDetailsBuilder.setParallelism(functionConfig.getParallelism());
    } else {
        functionDetailsBuilder.setParallelism(1);
    }
    // use default resources if resources not set
    Resources resources = Resources.mergeWithDefault(functionConfig.getResources());
    Function.Resources.Builder bldr = Function.Resources.newBuilder();
    bldr.setCpu(resources.getCpu());
    bldr.setRam(resources.getRam());
    bldr.setDisk(resources.getDisk());
    functionDetailsBuilder.setResources(bldr);
    if (!StringUtils.isEmpty(functionConfig.getRuntimeFlags())) {
        functionDetailsBuilder.setRuntimeFlags(functionConfig.getRuntimeFlags());
    }
    functionDetailsBuilder.setComponentType(FunctionDetails.ComponentType.FUNCTION);
    if (!StringUtils.isEmpty(functionConfig.getCustomRuntimeOptions())) {
        functionDetailsBuilder.setCustomRuntimeOptions(functionConfig.getCustomRuntimeOptions());
    }
    if (isBuiltin) {
        String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
        functionDetailsBuilder.setBuiltin(builtin);
    }
    return functionDetailsBuilder.build();
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) TypeToken(com.google.gson.reflect.TypeToken) TopicName(org.apache.pulsar.common.naming.TopicName) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) StringUtils.isNotEmpty(org.apache.commons.lang.StringUtils.isNotEmpty) HashMap(java.util.HashMap) Resources(org.apache.pulsar.common.functions.Resources) ClassLoaderUtils.loadJar(org.apache.pulsar.common.util.ClassLoaderUtils.loadJar) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) StringUtils.isBlank(org.apache.commons.lang.StringUtils.isBlank) FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails) Gson(com.google.gson.Gson) Map(java.util.Map) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) LinkedList(java.util.LinkedList) StringUtils.isEmpty(org.apache.commons.lang3.StringUtils.isEmpty) FunctionCommon.convertFromFunctionDetailsSubscriptionPosition(org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition) Function(org.apache.pulsar.functions.proto.Function) WindowConfig(org.apache.pulsar.common.functions.WindowConfig) BUILTIN(org.apache.pulsar.common.functions.Utils.BUILTIN) MalformedURLException(java.net.MalformedURLException) Collection(java.util.Collection) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) StringUtils.isNotBlank(org.apache.commons.lang.StringUtils.isNotBlank) File(java.io.File) ProducerConfig(org.apache.pulsar.common.functions.ProducerConfig) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Type(java.lang.reflect.Type) HashMap(java.util.HashMap) Gson(com.google.gson.Gson) Function(org.apache.pulsar.functions.proto.Function) ProducerConfig(org.apache.pulsar.common.functions.ProducerConfig) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) WindowConfig(org.apache.pulsar.common.functions.WindowConfig) FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails) Resources(org.apache.pulsar.common.functions.Resources)

Example 3 with BUILTIN

use of org.apache.pulsar.common.functions.Utils.BUILTIN in project pulsar by yahoo.

the class FunctionConfigUtils method convert.

public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) throws IllegalArgumentException {
    boolean isBuiltin = !org.apache.commons.lang3.StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN);
    Class<?>[] typeArgs = null;
    if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
        if (classLoader != null) {
            try {
                typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader);
            } catch (ClassNotFoundException | NoClassDefFoundError e) {
                throw new IllegalArgumentException(String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
            }
        }
    }
    FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
    // Setup source
    Function.SourceSpec.Builder sourceSpecBuilder = Function.SourceSpec.newBuilder();
    if (functionConfig.getInputs() != null) {
        functionConfig.getInputs().forEach((topicName -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setIsRegexPattern(false).build());
        }));
    }
    if (functionConfig.getTopicsPattern() != null && !functionConfig.getTopicsPattern().isEmpty()) {
        sourceSpecBuilder.putInputSpecs(functionConfig.getTopicsPattern(), Function.ConsumerSpec.newBuilder().setIsRegexPattern(true).build());
    }
    if (functionConfig.getCustomSerdeInputs() != null) {
        functionConfig.getCustomSerdeInputs().forEach((topicName, serdeClassName) -> {
            sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSerdeClassName(serdeClassName).setIsRegexPattern(false).build());
        });
    }
    if (functionConfig.getCustomSchemaInputs() != null) {
        functionConfig.getCustomSchemaInputs().forEach((topicName, conf) -> {
            try {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSchemaType(consumerConfig.getSchemaType()).putAllSchemaProperties(consumerConfig.getSchemaProperties()).putAllConsumerProperties(consumerConfig.getConsumerProperties()).setIsRegexPattern(false).build());
            } catch (JsonProcessingException e) {
                throw new IllegalArgumentException(String.format("Incorrect custom schema inputs,Topic %s ", topicName));
            }
        });
    }
    if (functionConfig.getInputSpecs() != null) {
        functionConfig.getInputSpecs().forEach((topicName, consumerConf) -> {
            Function.ConsumerSpec.Builder bldr = Function.ConsumerSpec.newBuilder().setIsRegexPattern(consumerConf.isRegexPattern());
            if (isNotBlank(consumerConf.getSchemaType())) {
                bldr.setSchemaType(consumerConf.getSchemaType());
            } else if (isNotBlank(consumerConf.getSerdeClassName())) {
                bldr.setSerdeClassName(consumerConf.getSerdeClassName());
            }
            if (consumerConf.getReceiverQueueSize() != null) {
                bldr.setReceiverQueueSize(Function.ConsumerSpec.ReceiverQueueSize.newBuilder().setValue(consumerConf.getReceiverQueueSize()).build());
            }
            if (consumerConf.getSchemaProperties() != null) {
                bldr.putAllSchemaProperties(consumerConf.getSchemaProperties());
            }
            if (consumerConf.getCryptoConfig() != null) {
                bldr.setCryptoSpec(CryptoUtils.convert(consumerConf.getCryptoConfig()));
            }
            bldr.putAllConsumerProperties(consumerConf.getConsumerProperties());
            bldr.setPoolMessages(consumerConf.isPoolMessages());
            sourceSpecBuilder.putInputSpecs(topicName, bldr.build());
        });
    }
    // Set subscription type
    Function.SubscriptionType subType;
    if ((functionConfig.getRetainOrdering() != null && functionConfig.getRetainOrdering()) || FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE.equals(functionConfig.getProcessingGuarantees())) {
        subType = Function.SubscriptionType.FAILOVER;
    } else if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering()) {
        subType = Function.SubscriptionType.KEY_SHARED;
    } else {
        subType = Function.SubscriptionType.SHARED;
    }
    sourceSpecBuilder.setSubscriptionType(subType);
    // Set subscription name
    if (isNotBlank(functionConfig.getSubName())) {
        sourceSpecBuilder.setSubscriptionName(functionConfig.getSubName());
    }
    // Set subscription position
    if (functionConfig.getSubscriptionPosition() != null) {
        Function.SubscriptionPosition subPosition = null;
        if (SubscriptionInitialPosition.Earliest == functionConfig.getSubscriptionPosition()) {
            subPosition = Function.SubscriptionPosition.EARLIEST;
        } else {
            subPosition = Function.SubscriptionPosition.LATEST;
        }
        sourceSpecBuilder.setSubscriptionPosition(subPosition);
    }
    if (typeArgs != null) {
        sourceSpecBuilder.setTypeClassName(typeArgs[0].getName());
    }
    if (functionConfig.getTimeoutMs() != null) {
        sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs());
        // We use negative acks for fast tracking failures
        sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(functionConfig.getTimeoutMs());
    }
    if (functionConfig.getCleanupSubscription() != null) {
        sourceSpecBuilder.setCleanupSubscription(functionConfig.getCleanupSubscription());
    } else {
        sourceSpecBuilder.setCleanupSubscription(true);
    }
    functionDetailsBuilder.setSource(sourceSpecBuilder);
    // Setup sink
    Function.SinkSpec.Builder sinkSpecBuilder = Function.SinkSpec.newBuilder();
    if (functionConfig.getOutput() != null) {
        sinkSpecBuilder.setTopic(functionConfig.getOutput());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSerdeClassName())) {
        sinkSpecBuilder.setSerDeClassName(functionConfig.getOutputSerdeClassName());
    }
    if (!StringUtils.isBlank(functionConfig.getOutputSchemaType())) {
        sinkSpecBuilder.setSchemaType(functionConfig.getOutputSchemaType());
    }
    if (functionConfig.getForwardSourceMessageProperty() == Boolean.TRUE) {
        sinkSpecBuilder.setForwardSourceMessageProperty(functionConfig.getForwardSourceMessageProperty());
    }
    if (functionConfig.getCustomSchemaOutputs() != null && functionConfig.getOutput() != null) {
        String conf = functionConfig.getCustomSchemaOutputs().get(functionConfig.getOutput());
        try {
            if (StringUtils.isNotEmpty(conf)) {
                ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
                sinkSpecBuilder.putAllSchemaProperties(consumerConfig.getSchemaProperties());
                sinkSpecBuilder.putAllConsumerProperties(consumerConfig.getConsumerProperties());
            }
        } catch (JsonProcessingException e) {
            throw new IllegalArgumentException(String.format("Incorrect custom schema outputs,Topic %s ", functionConfig.getOutput()));
        }
    }
    if (typeArgs != null) {
        sinkSpecBuilder.setTypeClassName(typeArgs[1].getName());
    }
    if (functionConfig.getProducerConfig() != null) {
        ProducerConfig producerConf = functionConfig.getProducerConfig();
        Function.ProducerSpec.Builder pbldr = Function.ProducerSpec.newBuilder();
        if (producerConf.getMaxPendingMessages() != null) {
            pbldr.setMaxPendingMessages(producerConf.getMaxPendingMessages());
        }
        if (producerConf.getMaxPendingMessagesAcrossPartitions() != null) {
            pbldr.setMaxPendingMessagesAcrossPartitions(producerConf.getMaxPendingMessagesAcrossPartitions());
        }
        if (producerConf.getUseThreadLocalProducers() != null) {
            pbldr.setUseThreadLocalProducers(producerConf.getUseThreadLocalProducers());
        }
        if (producerConf.getCryptoConfig() != null) {
            pbldr.setCryptoSpec(CryptoUtils.convert(producerConf.getCryptoConfig()));
        }
        if (producerConf.getBatchBuilder() != null) {
            pbldr.setBatchBuilder(producerConf.getBatchBuilder());
        }
        sinkSpecBuilder.setProducerSpec(pbldr.build());
    }
    functionDetailsBuilder.setSink(sinkSpecBuilder);
    if (functionConfig.getTenant() != null) {
        functionDetailsBuilder.setTenant(functionConfig.getTenant());
    }
    if (functionConfig.getNamespace() != null) {
        functionDetailsBuilder.setNamespace(functionConfig.getNamespace());
    }
    if (functionConfig.getName() != null) {
        functionDetailsBuilder.setName(functionConfig.getName());
    }
    if (functionConfig.getLogTopic() != null) {
        functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic());
    }
    if (functionConfig.getRuntime() != null) {
        functionDetailsBuilder.setRuntime(FunctionCommon.convertRuntime(functionConfig.getRuntime()));
    }
    if (functionConfig.getProcessingGuarantees() != null) {
        functionDetailsBuilder.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionConfig.getProcessingGuarantees()));
    }
    if (functionConfig.getRetainKeyOrdering() != null) {
        functionDetailsBuilder.setRetainKeyOrdering(functionConfig.getRetainKeyOrdering());
    }
    if (functionConfig.getRetainOrdering() != null) {
        functionDetailsBuilder.setRetainOrdering(functionConfig.getRetainOrdering());
    }
    if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0) {
        Function.RetryDetails.Builder retryBuilder = Function.RetryDetails.newBuilder();
        retryBuilder.setMaxMessageRetries(functionConfig.getMaxMessageRetries());
        if (isNotEmpty(functionConfig.getDeadLetterTopic())) {
            retryBuilder.setDeadLetterTopic(functionConfig.getDeadLetterTopic());
        }
        functionDetailsBuilder.setRetryDetails(retryBuilder);
    }
    Map<String, Object> configs = new HashMap<>();
    if (functionConfig.getUserConfig() != null) {
        configs.putAll(functionConfig.getUserConfig());
    }
    // windowing related
    WindowConfig windowConfig = functionConfig.getWindowConfig();
    if (windowConfig != null) {
        windowConfig.setActualWindowFunctionClassName(functionConfig.getClassName());
        configs.put(WindowConfig.WINDOW_CONFIG_KEY, windowConfig);
        // set class name to window function executor
        functionDetailsBuilder.setClassName("org.apache.pulsar.functions.windowing.WindowFunctionExecutor");
    } else {
        if (functionConfig.getClassName() != null) {
            functionDetailsBuilder.setClassName(functionConfig.getClassName());
        }
    }
    if (!configs.isEmpty()) {
        functionDetailsBuilder.setUserConfig(new Gson().toJson(configs));
    }
    if (functionConfig.getSecrets() != null && !functionConfig.getSecrets().isEmpty()) {
        functionDetailsBuilder.setSecretsMap(new Gson().toJson(functionConfig.getSecrets()));
    }
    if (functionConfig.getAutoAck() != null) {
        functionDetailsBuilder.setAutoAck(functionConfig.getAutoAck());
    } else {
        functionDetailsBuilder.setAutoAck(true);
    }
    if (functionConfig.getParallelism() != null) {
        functionDetailsBuilder.setParallelism(functionConfig.getParallelism());
    } else {
        functionDetailsBuilder.setParallelism(1);
    }
    // use default resources if resources not set
    Resources resources = Resources.mergeWithDefault(functionConfig.getResources());
    Function.Resources.Builder bldr = Function.Resources.newBuilder();
    bldr.setCpu(resources.getCpu());
    bldr.setRam(resources.getRam());
    bldr.setDisk(resources.getDisk());
    functionDetailsBuilder.setResources(bldr);
    if (!StringUtils.isEmpty(functionConfig.getRuntimeFlags())) {
        functionDetailsBuilder.setRuntimeFlags(functionConfig.getRuntimeFlags());
    }
    functionDetailsBuilder.setComponentType(FunctionDetails.ComponentType.FUNCTION);
    if (!StringUtils.isEmpty(functionConfig.getCustomRuntimeOptions())) {
        functionDetailsBuilder.setCustomRuntimeOptions(functionConfig.getCustomRuntimeOptions());
    }
    if (isBuiltin) {
        String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
        functionDetailsBuilder.setBuiltin(builtin);
    }
    return functionDetailsBuilder.build();
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) TypeToken(com.google.gson.reflect.TypeToken) TopicName(org.apache.pulsar.common.naming.TopicName) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) StringUtils.isNotEmpty(org.apache.commons.lang.StringUtils.isNotEmpty) HashMap(java.util.HashMap) Resources(org.apache.pulsar.common.functions.Resources) ClassLoaderUtils.loadJar(org.apache.pulsar.common.util.ClassLoaderUtils.loadJar) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) StringUtils.isBlank(org.apache.commons.lang.StringUtils.isBlank) FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails) Gson(com.google.gson.Gson) Map(java.util.Map) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) LinkedList(java.util.LinkedList) StringUtils.isEmpty(org.apache.commons.lang3.StringUtils.isEmpty) FunctionCommon.convertFromFunctionDetailsSubscriptionPosition(org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition) Function(org.apache.pulsar.functions.proto.Function) WindowConfig(org.apache.pulsar.common.functions.WindowConfig) BUILTIN(org.apache.pulsar.common.functions.Utils.BUILTIN) MalformedURLException(java.net.MalformedURLException) Collection(java.util.Collection) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) StringUtils.isNotBlank(org.apache.commons.lang.StringUtils.isNotBlank) File(java.io.File) ProducerConfig(org.apache.pulsar.common.functions.ProducerConfig) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Type(java.lang.reflect.Type) HashMap(java.util.HashMap) Gson(com.google.gson.Gson) Function(org.apache.pulsar.functions.proto.Function) ProducerConfig(org.apache.pulsar.common.functions.ProducerConfig) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) WindowConfig(org.apache.pulsar.common.functions.WindowConfig) FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails) Resources(org.apache.pulsar.common.functions.Resources)

Aggregations

JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 Gson (com.google.gson.Gson)3 TypeToken (com.google.gson.reflect.TypeToken)3 File (java.io.File)3 Type (java.lang.reflect.Type)3 MalformedURLException (java.net.MalformedURLException)3 Collection (java.util.Collection)3 HashMap (java.util.HashMap)3 LinkedList (java.util.LinkedList)3 List (java.util.List)3 Map (java.util.Map)3 Slf4j (lombok.extern.slf4j.Slf4j)3 StringUtils (org.apache.commons.lang.StringUtils)3 StringUtils.isBlank (org.apache.commons.lang.StringUtils.isBlank)3 StringUtils.isNotBlank (org.apache.commons.lang.StringUtils.isNotBlank)3 StringUtils.isNotEmpty (org.apache.commons.lang.StringUtils.isNotEmpty)3 StringUtils.isEmpty (org.apache.commons.lang3.StringUtils.isEmpty)3 SubscriptionInitialPosition (org.apache.pulsar.client.api.SubscriptionInitialPosition)3 ConsumerConfig (org.apache.pulsar.common.functions.ConsumerConfig)3