use of org.apache.pulsar.common.functions.WindowConfig in project pulsar by apache.
the class FunctionConfigUtilsTest method createFunctionConfig.
private FunctionConfig createFunctionConfig() {
FunctionConfig functionConfig = new FunctionConfig();
functionConfig.setTenant("test-tenant");
functionConfig.setNamespace("test-namespace");
functionConfig.setName("test-function");
functionConfig.setParallelism(1);
functionConfig.setClassName(IdentityFunction.class.getName());
Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build());
functionConfig.setInputSpecs(inputSpecs);
functionConfig.setOutput("test-output");
functionConfig.setOutputSerdeClassName("test-serde");
functionConfig.setOutputSchemaType("json");
functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
functionConfig.setRetainOrdering(false);
functionConfig.setRetainKeyOrdering(false);
functionConfig.setSubscriptionPosition(SubscriptionInitialPosition.Earliest);
functionConfig.setBatchBuilder("DEFAULT");
functionConfig.setForwardSourceMessageProperty(false);
functionConfig.setUserConfig(new HashMap<>());
functionConfig.setAutoAck(true);
functionConfig.setTimeoutMs(2000l);
functionConfig.setWindowConfig(new WindowConfig().setWindowLengthCount(10));
functionConfig.setCleanupSubscription(true);
functionConfig.setRuntimeFlags("-Dfoo=bar");
return functionConfig;
}
use of org.apache.pulsar.common.functions.WindowConfig in project pulsar by apache.
the class WindowConfigUtilsTest method testSettingSlidingTimeWindow.
@Test
public void testSettingSlidingTimeWindow() throws Exception {
final Object[][] args = new Object[][] { { -1L, 10L }, { 10L, -1L }, { 0L, 10L }, { 10L, 0L }, { 0L, 0L }, { -1L, -1L }, { 5L, 10L }, { 1L, 1L }, { 10L, 5L }, { 100L, 10L }, { 100L, 100L }, { 200L, 100L }, { 500L, 100L }, { null, null }, { null, 1L }, { 1L, null }, { null, -1L }, { -1L, null } };
for (Object[] arg : args) {
Object arg0 = arg[0];
Object arg1 = arg[1];
try {
Long windowLengthDuration = null;
if (arg0 != null) {
windowLengthDuration = (Long) arg0;
}
Long slidingIntervalDuration = null;
if (arg1 != null) {
slidingIntervalDuration = (Long) arg1;
}
WindowConfig windowConfig = new WindowConfig();
windowConfig.setWindowLengthDurationMs(windowLengthDuration);
windowConfig.setSlidingIntervalDurationMs(slidingIntervalDuration);
WindowConfigUtils.validate(windowConfig);
if (arg0 == null) {
fail(String.format("Window length cannot be null -- " + "windowLengthCount: %s slidingIntervalCount: %s", arg0, arg1));
}
if ((Long) arg0 <= 0) {
fail(String.format("Window length cannot be zero or less -- " + "windowLengthCount: %s slidingIntervalCount: %s", arg0, arg1));
}
if (arg1 != null && (Long) arg1 <= 0) {
fail(String.format("Sliding interval length cannot be zero or less -- " + "windowLengthCount: %s slidingIntervalCount: %s", arg0, arg1));
}
} catch (IllegalArgumentException e) {
if (arg0 != null && arg1 != null && (Long) arg0 > 0 && (Long) arg1 > 0) {
fail(String.format("Exception: %s thrown on valid input -- windowLengthDuration: %s " + "slidingIntervalDuration: %s", e.getMessage(), arg0, arg1));
}
}
}
}
use of org.apache.pulsar.common.functions.WindowConfig in project pulsar by apache.
the class WindowConfigUtilsTest method testSettingLagTime.
@Test
public void testSettingLagTime() throws Exception {
final Object[] args = new Object[] { -1L, 0L, 1L, 2L, 5L, 10L, null };
for (Object arg : args) {
Object arg0 = arg;
try {
Long maxLagMs = null;
if (arg0 != null) {
maxLagMs = (Long) arg0;
}
WindowConfig windowConfig = new WindowConfig();
windowConfig.setWindowLengthCount(1);
windowConfig.setSlidingIntervalCount(1);
windowConfig.setMaxLagMs(maxLagMs);
windowConfig.setTimestampExtractorClassName("SomeClass");
WindowConfigUtils.validate(windowConfig);
if (arg0 != null && (Long) arg0 < 0) {
fail(String.format("Window lag cannot be less than zero -- lagTime: %s", arg0));
}
} catch (IllegalArgumentException e) {
if (arg0 != null && (Long) arg0 > 0) {
fail(String.format("Exception: %s thrown on valid input -- lagTime: %s", e.getMessage(), arg0));
}
}
}
}
use of org.apache.pulsar.common.functions.WindowConfig in project pulsar by apache.
the class FunctionConfigUtils method convert.
public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) throws IllegalArgumentException {
boolean isBuiltin = !org.apache.commons.lang3.StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN);
Class<?>[] typeArgs = null;
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
if (classLoader != null) {
try {
typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader);
} catch (ClassNotFoundException | NoClassDefFoundError e) {
throw new IllegalArgumentException(String.format("Function class %s must be in class path", functionConfig.getClassName()), e);
}
}
}
FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder();
// Setup source
Function.SourceSpec.Builder sourceSpecBuilder = Function.SourceSpec.newBuilder();
if (functionConfig.getInputs() != null) {
functionConfig.getInputs().forEach((topicName -> {
sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setIsRegexPattern(false).build());
}));
}
if (functionConfig.getTopicsPattern() != null && !functionConfig.getTopicsPattern().isEmpty()) {
sourceSpecBuilder.putInputSpecs(functionConfig.getTopicsPattern(), Function.ConsumerSpec.newBuilder().setIsRegexPattern(true).build());
}
if (functionConfig.getCustomSerdeInputs() != null) {
functionConfig.getCustomSerdeInputs().forEach((topicName, serdeClassName) -> {
sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSerdeClassName(serdeClassName).setIsRegexPattern(false).build());
});
}
if (functionConfig.getCustomSchemaInputs() != null) {
functionConfig.getCustomSchemaInputs().forEach((topicName, conf) -> {
try {
ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
sourceSpecBuilder.putInputSpecs(topicName, Function.ConsumerSpec.newBuilder().setSchemaType(consumerConfig.getSchemaType()).putAllSchemaProperties(consumerConfig.getSchemaProperties()).putAllConsumerProperties(consumerConfig.getConsumerProperties()).setIsRegexPattern(false).build());
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(String.format("Incorrect custom schema inputs,Topic %s ", topicName));
}
});
}
if (functionConfig.getInputSpecs() != null) {
functionConfig.getInputSpecs().forEach((topicName, consumerConf) -> {
Function.ConsumerSpec.Builder bldr = Function.ConsumerSpec.newBuilder().setIsRegexPattern(consumerConf.isRegexPattern());
if (isNotBlank(consumerConf.getSchemaType())) {
bldr.setSchemaType(consumerConf.getSchemaType());
} else if (isNotBlank(consumerConf.getSerdeClassName())) {
bldr.setSerdeClassName(consumerConf.getSerdeClassName());
}
if (consumerConf.getReceiverQueueSize() != null) {
bldr.setReceiverQueueSize(Function.ConsumerSpec.ReceiverQueueSize.newBuilder().setValue(consumerConf.getReceiverQueueSize()).build());
}
if (consumerConf.getSchemaProperties() != null) {
bldr.putAllSchemaProperties(consumerConf.getSchemaProperties());
}
if (consumerConf.getCryptoConfig() != null) {
bldr.setCryptoSpec(CryptoUtils.convert(consumerConf.getCryptoConfig()));
}
bldr.putAllConsumerProperties(consumerConf.getConsumerProperties());
bldr.setPoolMessages(consumerConf.isPoolMessages());
sourceSpecBuilder.putInputSpecs(topicName, bldr.build());
});
}
// Set subscription type
Function.SubscriptionType subType;
if ((functionConfig.getRetainOrdering() != null && functionConfig.getRetainOrdering()) || FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE.equals(functionConfig.getProcessingGuarantees())) {
subType = Function.SubscriptionType.FAILOVER;
} else if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering()) {
subType = Function.SubscriptionType.KEY_SHARED;
} else {
subType = Function.SubscriptionType.SHARED;
}
sourceSpecBuilder.setSubscriptionType(subType);
// Set subscription name
if (isNotBlank(functionConfig.getSubName())) {
sourceSpecBuilder.setSubscriptionName(functionConfig.getSubName());
}
// Set subscription position
if (functionConfig.getSubscriptionPosition() != null) {
Function.SubscriptionPosition subPosition = null;
if (SubscriptionInitialPosition.Earliest == functionConfig.getSubscriptionPosition()) {
subPosition = Function.SubscriptionPosition.EARLIEST;
} else {
subPosition = Function.SubscriptionPosition.LATEST;
}
sourceSpecBuilder.setSubscriptionPosition(subPosition);
}
if (typeArgs != null) {
sourceSpecBuilder.setTypeClassName(typeArgs[0].getName());
}
if (functionConfig.getTimeoutMs() != null) {
sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs());
// We use negative acks for fast tracking failures
sourceSpecBuilder.setNegativeAckRedeliveryDelayMs(functionConfig.getTimeoutMs());
}
if (functionConfig.getCleanupSubscription() != null) {
sourceSpecBuilder.setCleanupSubscription(functionConfig.getCleanupSubscription());
} else {
sourceSpecBuilder.setCleanupSubscription(true);
}
functionDetailsBuilder.setSource(sourceSpecBuilder);
// Setup sink
Function.SinkSpec.Builder sinkSpecBuilder = Function.SinkSpec.newBuilder();
if (functionConfig.getOutput() != null) {
sinkSpecBuilder.setTopic(functionConfig.getOutput());
}
if (!StringUtils.isBlank(functionConfig.getOutputSerdeClassName())) {
sinkSpecBuilder.setSerDeClassName(functionConfig.getOutputSerdeClassName());
}
if (!StringUtils.isBlank(functionConfig.getOutputSchemaType())) {
sinkSpecBuilder.setSchemaType(functionConfig.getOutputSchemaType());
}
if (functionConfig.getForwardSourceMessageProperty() == Boolean.TRUE) {
sinkSpecBuilder.setForwardSourceMessageProperty(functionConfig.getForwardSourceMessageProperty());
}
if (functionConfig.getCustomSchemaOutputs() != null && functionConfig.getOutput() != null) {
String conf = functionConfig.getCustomSchemaOutputs().get(functionConfig.getOutput());
try {
if (StringUtils.isNotEmpty(conf)) {
ConsumerConfig consumerConfig = OBJECT_MAPPER.readValue(conf, ConsumerConfig.class);
sinkSpecBuilder.putAllSchemaProperties(consumerConfig.getSchemaProperties());
sinkSpecBuilder.putAllConsumerProperties(consumerConfig.getConsumerProperties());
}
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(String.format("Incorrect custom schema outputs,Topic %s ", functionConfig.getOutput()));
}
}
if (typeArgs != null) {
sinkSpecBuilder.setTypeClassName(typeArgs[1].getName());
}
if (functionConfig.getProducerConfig() != null) {
ProducerConfig producerConf = functionConfig.getProducerConfig();
Function.ProducerSpec.Builder pbldr = Function.ProducerSpec.newBuilder();
if (producerConf.getMaxPendingMessages() != null) {
pbldr.setMaxPendingMessages(producerConf.getMaxPendingMessages());
}
if (producerConf.getMaxPendingMessagesAcrossPartitions() != null) {
pbldr.setMaxPendingMessagesAcrossPartitions(producerConf.getMaxPendingMessagesAcrossPartitions());
}
if (producerConf.getUseThreadLocalProducers() != null) {
pbldr.setUseThreadLocalProducers(producerConf.getUseThreadLocalProducers());
}
if (producerConf.getCryptoConfig() != null) {
pbldr.setCryptoSpec(CryptoUtils.convert(producerConf.getCryptoConfig()));
}
if (producerConf.getBatchBuilder() != null) {
pbldr.setBatchBuilder(producerConf.getBatchBuilder());
}
sinkSpecBuilder.setProducerSpec(pbldr.build());
}
functionDetailsBuilder.setSink(sinkSpecBuilder);
if (functionConfig.getTenant() != null) {
functionDetailsBuilder.setTenant(functionConfig.getTenant());
}
if (functionConfig.getNamespace() != null) {
functionDetailsBuilder.setNamespace(functionConfig.getNamespace());
}
if (functionConfig.getName() != null) {
functionDetailsBuilder.setName(functionConfig.getName());
}
if (functionConfig.getLogTopic() != null) {
functionDetailsBuilder.setLogTopic(functionConfig.getLogTopic());
}
if (functionConfig.getRuntime() != null) {
functionDetailsBuilder.setRuntime(FunctionCommon.convertRuntime(functionConfig.getRuntime()));
}
if (functionConfig.getProcessingGuarantees() != null) {
functionDetailsBuilder.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionConfig.getProcessingGuarantees()));
}
if (functionConfig.getRetainKeyOrdering() != null) {
functionDetailsBuilder.setRetainKeyOrdering(functionConfig.getRetainKeyOrdering());
}
if (functionConfig.getRetainOrdering() != null) {
functionDetailsBuilder.setRetainOrdering(functionConfig.getRetainOrdering());
}
if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0) {
Function.RetryDetails.Builder retryBuilder = Function.RetryDetails.newBuilder();
retryBuilder.setMaxMessageRetries(functionConfig.getMaxMessageRetries());
if (isNotEmpty(functionConfig.getDeadLetterTopic())) {
retryBuilder.setDeadLetterTopic(functionConfig.getDeadLetterTopic());
}
functionDetailsBuilder.setRetryDetails(retryBuilder);
}
Map<String, Object> configs = new HashMap<>();
if (functionConfig.getUserConfig() != null) {
configs.putAll(functionConfig.getUserConfig());
}
// windowing related
WindowConfig windowConfig = functionConfig.getWindowConfig();
if (windowConfig != null) {
windowConfig.setActualWindowFunctionClassName(functionConfig.getClassName());
configs.put(WindowConfig.WINDOW_CONFIG_KEY, windowConfig);
// set class name to window function executor
functionDetailsBuilder.setClassName("org.apache.pulsar.functions.windowing.WindowFunctionExecutor");
} else {
if (functionConfig.getClassName() != null) {
functionDetailsBuilder.setClassName(functionConfig.getClassName());
}
}
if (!configs.isEmpty()) {
functionDetailsBuilder.setUserConfig(new Gson().toJson(configs));
}
if (functionConfig.getSecrets() != null && !functionConfig.getSecrets().isEmpty()) {
functionDetailsBuilder.setSecretsMap(new Gson().toJson(functionConfig.getSecrets()));
}
if (functionConfig.getAutoAck() != null) {
functionDetailsBuilder.setAutoAck(functionConfig.getAutoAck());
} else {
functionDetailsBuilder.setAutoAck(true);
}
if (functionConfig.getParallelism() != null) {
functionDetailsBuilder.setParallelism(functionConfig.getParallelism());
} else {
functionDetailsBuilder.setParallelism(1);
}
// use default resources if resources not set
Resources resources = Resources.mergeWithDefault(functionConfig.getResources());
Function.Resources.Builder bldr = Function.Resources.newBuilder();
bldr.setCpu(resources.getCpu());
bldr.setRam(resources.getRam());
bldr.setDisk(resources.getDisk());
functionDetailsBuilder.setResources(bldr);
if (!StringUtils.isEmpty(functionConfig.getRuntimeFlags())) {
functionDetailsBuilder.setRuntimeFlags(functionConfig.getRuntimeFlags());
}
functionDetailsBuilder.setComponentType(FunctionDetails.ComponentType.FUNCTION);
if (!StringUtils.isEmpty(functionConfig.getCustomRuntimeOptions())) {
functionDetailsBuilder.setCustomRuntimeOptions(functionConfig.getCustomRuntimeOptions());
}
if (isBuiltin) {
String builtin = functionConfig.getJar().replaceFirst("^builtin://", "");
functionDetailsBuilder.setBuiltin(builtin);
}
return functionDetailsBuilder.build();
}
use of org.apache.pulsar.common.functions.WindowConfig in project pulsar by apache.
the class FunctionConfigUtils method doCommonChecks.
private static void doCommonChecks(FunctionConfig functionConfig) {
if (isEmpty(functionConfig.getTenant())) {
throw new IllegalArgumentException("Function tenant cannot be null");
}
if (isEmpty(functionConfig.getNamespace())) {
throw new IllegalArgumentException("Function namespace cannot be null");
}
if (isEmpty(functionConfig.getName())) {
throw new IllegalArgumentException("Function name cannot be null");
}
// go doesn't need className
if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON || functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
if (isEmpty(functionConfig.getClassName())) {
throw new IllegalArgumentException("Function classname cannot be null");
}
}
Collection<String> allInputTopics = collectAllInputTopics(functionConfig);
if (allInputTopics.isEmpty()) {
throw new IllegalArgumentException("No input topic(s) specified for the function");
}
for (String topic : allInputTopics) {
if (!TopicName.isValid(topic)) {
throw new IllegalArgumentException(String.format("Input topic %s is invalid", topic));
}
}
if (!isEmpty(functionConfig.getOutput())) {
if (!TopicName.isValid(functionConfig.getOutput())) {
throw new IllegalArgumentException(String.format("Output topic %s is invalid", functionConfig.getOutput()));
}
}
if (!isEmpty(functionConfig.getLogTopic())) {
if (!TopicName.isValid(functionConfig.getLogTopic())) {
throw new IllegalArgumentException(String.format("LogTopic topic %s is invalid", functionConfig.getLogTopic()));
}
}
if (!isEmpty(functionConfig.getDeadLetterTopic())) {
if (!TopicName.isValid(functionConfig.getDeadLetterTopic())) {
throw new IllegalArgumentException(String.format("DeadLetter topic %s is invalid", functionConfig.getDeadLetterTopic()));
}
}
if (functionConfig.getParallelism() != null && functionConfig.getParallelism() <= 0) {
throw new IllegalArgumentException("Function parallelism must be a positive number");
}
// Ensure that topics aren't being used as both input and output
verifyNoTopicClash(allInputTopics, functionConfig.getOutput());
WindowConfig windowConfig = functionConfig.getWindowConfig();
if (windowConfig != null) {
// for acking and not the function framework
if (functionConfig.getAutoAck() != null && functionConfig.getAutoAck()) {
throw new IllegalArgumentException("Cannot enable auto ack when using windowing functionality");
}
WindowConfigUtils.validate(windowConfig);
}
if (functionConfig.getResources() != null) {
ResourceConfigUtils.validate(functionConfig.getResources());
}
if (functionConfig.getTimeoutMs() != null && functionConfig.getTimeoutMs() <= 0) {
throw new IllegalArgumentException("Function timeout must be a positive number");
}
if (functionConfig.getTimeoutMs() != null && functionConfig.getProcessingGuarantees() != null && functionConfig.getProcessingGuarantees() != FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE) {
throw new IllegalArgumentException("Message timeout can only be specified with processing guarantee is " + FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE.name());
}
if (functionConfig.getMaxMessageRetries() != null && functionConfig.getMaxMessageRetries() >= 0 && functionConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
throw new IllegalArgumentException("MaxMessageRetries and Effectively once don't gel well");
}
if ((functionConfig.getMaxMessageRetries() == null || functionConfig.getMaxMessageRetries() < 0) && !org.apache.commons.lang3.StringUtils.isEmpty(functionConfig.getDeadLetterTopic())) {
throw new IllegalArgumentException("Dead Letter Topic specified, however max retries is set to infinity");
}
if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering() && functionConfig.getProcessingGuarantees() != null && functionConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
throw new IllegalArgumentException("When effectively once processing guarantee is specified, retain Key ordering cannot be set");
}
if (functionConfig.getRetainKeyOrdering() != null && functionConfig.getRetainKeyOrdering() && functionConfig.getRetainOrdering() != null && functionConfig.getRetainOrdering()) {
throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set");
}
if (!isEmpty(functionConfig.getPy()) && !org.apache.pulsar.common.functions.Utils.isFunctionPackageUrlSupported(functionConfig.getPy()) && functionConfig.getPy().startsWith(BUILTIN)) {
if (!new File(functionConfig.getPy()).exists()) {
throw new IllegalArgumentException("The supplied python file does not exist");
}
}
if (!isEmpty(functionConfig.getGo()) && !org.apache.pulsar.common.functions.Utils.isFunctionPackageUrlSupported(functionConfig.getGo()) && functionConfig.getGo().startsWith(BUILTIN)) {
if (!new File(functionConfig.getGo()).exists()) {
throw new IllegalArgumentException("The supplied go file does not exist");
}
}
if (functionConfig.getInputSpecs() != null) {
functionConfig.getInputSpecs().forEach((topicName, conf) -> {
// receiver queue size should be >= 0
if (conf.getReceiverQueueSize() != null && conf.getReceiverQueueSize() < 0) {
throw new IllegalArgumentException("Receiver queue size should be >= zero");
}
if (conf.getCryptoConfig() != null && isBlank(conf.getCryptoConfig().getCryptoKeyReaderClassName())) {
throw new IllegalArgumentException("CryptoKeyReader class name required");
}
});
}
if (functionConfig.getProducerConfig() != null && functionConfig.getProducerConfig().getCryptoConfig() != null) {
if (isBlank(functionConfig.getProducerConfig().getCryptoConfig().getCryptoKeyReaderClassName())) {
throw new IllegalArgumentException("CryptoKeyReader class name required");
}
if (functionConfig.getProducerConfig().getCryptoConfig().getEncryptionKeys() == null || functionConfig.getProducerConfig().getCryptoConfig().getEncryptionKeys().length == 0) {
throw new IllegalArgumentException("Must provide encryption key name for crypto key reader");
}
}
}
Aggregations