use of org.apache.pulsar.functions.proto.Function.SourceSpec in project pulsar by apache.
the class FunctionActioner method fillSinkTypeClass.
private void fillSinkTypeClass(FunctionDetails.Builder functionDetails, ClassLoader narClassLoader, String className) throws ClassNotFoundException {
String typeArg = getSinkType(className, narClassLoader).getName();
SinkSpec.Builder sinkBuilder = SinkSpec.newBuilder(functionDetails.getSink());
sinkBuilder.setTypeClassName(typeArg);
functionDetails.setSink(sinkBuilder);
SourceSpec sourceSpec = functionDetails.getSource();
if (null == sourceSpec || StringUtils.isEmpty(sourceSpec.getTypeClassName())) {
SourceSpec.Builder sourceBuilder = SourceSpec.newBuilder(sourceSpec);
sourceBuilder.setTypeClassName(typeArg);
functionDetails.setSource(sourceBuilder);
}
}
use of org.apache.pulsar.functions.proto.Function.SourceSpec in project pulsar by apache.
the class FunctionActioner method getBuiltinArchive.
private File getBuiltinArchive(FunctionDetails.Builder functionDetails) throws IOException, ClassNotFoundException {
if (functionDetails.hasSource()) {
SourceSpec sourceSpec = functionDetails.getSource();
if (!StringUtils.isEmpty(sourceSpec.getBuiltin())) {
Connector connector = connectorsManager.getConnector(sourceSpec.getBuiltin());
File archive = connector.getArchivePath().toFile();
String sourceClass = connector.getConnectorDefinition().getSourceClass();
SourceSpec.Builder builder = SourceSpec.newBuilder(functionDetails.getSource());
builder.setClassName(sourceClass);
functionDetails.setSource(builder);
fillSourceTypeClass(functionDetails, connector.getClassLoader(), sourceClass);
return archive;
}
}
if (functionDetails.hasSink()) {
SinkSpec sinkSpec = functionDetails.getSink();
if (!StringUtils.isEmpty(sinkSpec.getBuiltin())) {
Connector connector = connectorsManager.getConnector(sinkSpec.getBuiltin());
File archive = connector.getArchivePath().toFile();
String sinkClass = connector.getConnectorDefinition().getSinkClass();
SinkSpec.Builder builder = SinkSpec.newBuilder(functionDetails.getSink());
builder.setClassName(sinkClass);
functionDetails.setSink(builder);
fillSinkTypeClass(functionDetails, connector.getClassLoader(), sinkClass);
return archive;
}
}
if (!StringUtils.isEmpty(functionDetails.getBuiltin())) {
return functionsManager.getFunctionArchive(functionDetails.getBuiltin()).toFile();
}
throw new IOException("Could not find built in archive definition");
}
use of org.apache.pulsar.functions.proto.Function.SourceSpec in project pulsar by apache.
the class JavaInstanceRunnable method setupInput.
private void setupInput(ContextImpl contextImpl) throws Exception {
SourceSpec sourceSpec = this.instanceConfig.getFunctionDetails().getSource();
Object object;
// If source classname is not set, we default pulsar source
if (sourceSpec.getClassName().isEmpty()) {
Map<String, ConsumerConfig> topicSchema = new TreeMap<>();
sourceSpec.getInputSpecsMap().forEach((topic, conf) -> {
ConsumerConfig consumerConfig = ConsumerConfig.builder().isRegexPattern(conf.getIsRegexPattern()).build();
if (conf.getSchemaType() != null && !conf.getSchemaType().isEmpty()) {
consumerConfig.setSchemaType(conf.getSchemaType());
} else if (conf.getSerdeClassName() != null && !conf.getSerdeClassName().isEmpty()) {
consumerConfig.setSerdeClassName(conf.getSerdeClassName());
}
consumerConfig.setSchemaProperties(conf.getSchemaPropertiesMap());
consumerConfig.setConsumerProperties(conf.getConsumerPropertiesMap());
if (conf.hasReceiverQueueSize()) {
consumerConfig.setReceiverQueueSize(conf.getReceiverQueueSize().getValue());
}
if (conf.hasCryptoSpec()) {
consumerConfig.setCryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
}
consumerConfig.setPoolMessages(conf.getPoolMessages());
topicSchema.put(topic, consumerConfig);
});
sourceSpec.getTopicsToSerDeClassNameMap().forEach((topic, serde) -> {
topicSchema.put(topic, ConsumerConfig.builder().serdeClassName(serde).isRegexPattern(false).build());
});
if (!StringUtils.isEmpty(sourceSpec.getTopicsPattern())) {
topicSchema.get(sourceSpec.getTopicsPattern()).setRegexPattern(true);
}
PulsarSourceConfig pulsarSourceConfig;
// we can use a single consumer to read
if (topicSchema.size() == 1) {
SingleConsumerPulsarSourceConfig singleConsumerPulsarSourceConfig = new SingleConsumerPulsarSourceConfig();
Map.Entry<String, ConsumerConfig> entry = topicSchema.entrySet().iterator().next();
singleConsumerPulsarSourceConfig.setTopic(entry.getKey());
singleConsumerPulsarSourceConfig.setConsumerConfig(entry.getValue());
pulsarSourceConfig = singleConsumerPulsarSourceConfig;
} else {
MultiConsumerPulsarSourceConfig multiConsumerPulsarSourceConfig = new MultiConsumerPulsarSourceConfig();
multiConsumerPulsarSourceConfig.setTopicSchema(topicSchema);
pulsarSourceConfig = multiConsumerPulsarSourceConfig;
}
pulsarSourceConfig.setSubscriptionName(StringUtils.isNotBlank(sourceSpec.getSubscriptionName()) ? sourceSpec.getSubscriptionName() : InstanceUtils.getDefaultSubscriptionName(instanceConfig.getFunctionDetails()));
pulsarSourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
pulsarSourceConfig.setSubscriptionPosition(convertFromFunctionDetailsSubscriptionPosition(sourceSpec.getSubscriptionPosition()));
checkNotNull(contextImpl.getSubscriptionType());
pulsarSourceConfig.setSubscriptionType(contextImpl.getSubscriptionType());
pulsarSourceConfig.setTypeClassName(sourceSpec.getTypeClassName());
if (sourceSpec.getTimeoutMs() > 0) {
pulsarSourceConfig.setTimeoutMs(sourceSpec.getTimeoutMs());
}
if (sourceSpec.getNegativeAckRedeliveryDelayMs() > 0) {
pulsarSourceConfig.setNegativeAckRedeliveryDelayMs(sourceSpec.getNegativeAckRedeliveryDelayMs());
}
if (this.instanceConfig.getFunctionDetails().hasRetryDetails()) {
pulsarSourceConfig.setMaxMessageRetries(this.instanceConfig.getFunctionDetails().getRetryDetails().getMaxMessageRetries());
pulsarSourceConfig.setDeadLetterTopic(this.instanceConfig.getFunctionDetails().getRetryDetails().getDeadLetterTopic());
}
// that require messages to be put into an immediate queue
if (pulsarSourceConfig instanceof SingleConsumerPulsarSourceConfig) {
object = new SingleConsumerPulsarSource(this.client, (SingleConsumerPulsarSourceConfig) pulsarSourceConfig, this.properties, this.functionClassLoader);
} else {
object = new MultiConsumerPulsarSource(this.client, (MultiConsumerPulsarSourceConfig) pulsarSourceConfig, this.properties, this.functionClassLoader);
}
} else {
// check if source is a batch source
if (sourceSpec.getClassName().equals(BatchSourceExecutor.class.getName())) {
object = Reflections.createInstance(sourceSpec.getClassName(), this.instanceClassLoader);
} else {
object = Reflections.createInstance(sourceSpec.getClassName(), this.functionClassLoader);
}
}
Class<?>[] typeArgs;
if (object instanceof Source) {
typeArgs = TypeResolver.resolveRawArguments(Source.class, object.getClass());
assert typeArgs.length > 0;
} else {
throw new RuntimeException("Source does not implement correct interface");
}
this.source = (Source<?>) object;
if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) {
Thread.currentThread().setContextClassLoader(this.functionClassLoader);
}
try {
if (sourceSpec.getConfigs().isEmpty()) {
this.source.open(new HashMap<>(), contextImpl);
} else {
this.source.open(ObjectMapperFactory.getThreadLocal().readValue(sourceSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
}), contextImpl);
}
if (this.source instanceof PulsarSource) {
contextImpl.setInputConsumers(((PulsarSource) this.source).getInputConsumers());
}
} catch (Exception e) {
log.error("Source open produced uncaught exception: ", e);
throw e;
} finally {
Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
}
}
use of org.apache.pulsar.functions.proto.Function.SourceSpec in project pulsar by apache.
the class SourceApiV3ResourceTest method testGetSourceSuccess.
@Test
public void testGetSourceSuccess() {
when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true);
SourceSpec sourceSpec = SourceSpec.newBuilder().setBuiltin("jdbc").build();
SinkSpec sinkSpec = SinkSpec.newBuilder().setTopic(outputTopic).setSerDeClassName(outputSerdeClassName).build();
FunctionDetails functionDetails = FunctionDetails.newBuilder().setClassName(IdentityFunction.class.getName()).setSink(sinkSpec).setName(source).setNamespace(namespace).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setRuntime(FunctionDetails.Runtime.JAVA).setAutoAck(true).setTenant(tenant).setParallelism(parallelism).setSource(sourceSpec).build();
FunctionMetaData metaData = FunctionMetaData.newBuilder().setCreateTime(System.currentTimeMillis()).setFunctionDetails(functionDetails).setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")).setVersion(1234).build();
when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(source))).thenReturn(metaData);
SourceConfig config = getDefaultSourceInfo();
assertEquals(SourceConfigUtils.convertFromDetails(functionDetails), config);
}
use of org.apache.pulsar.functions.proto.Function.SourceSpec in project pulsar by yahoo.
the class FunctionActioner method getBuiltinArchive.
private File getBuiltinArchive(FunctionDetails.Builder functionDetails) throws IOException, ClassNotFoundException {
if (functionDetails.hasSource()) {
SourceSpec sourceSpec = functionDetails.getSource();
if (!StringUtils.isEmpty(sourceSpec.getBuiltin())) {
Connector connector = connectorsManager.getConnector(sourceSpec.getBuiltin());
File archive = connector.getArchivePath().toFile();
String sourceClass = connector.getConnectorDefinition().getSourceClass();
SourceSpec.Builder builder = SourceSpec.newBuilder(functionDetails.getSource());
builder.setClassName(sourceClass);
functionDetails.setSource(builder);
fillSourceTypeClass(functionDetails, connector.getClassLoader(), sourceClass);
return archive;
}
}
if (functionDetails.hasSink()) {
SinkSpec sinkSpec = functionDetails.getSink();
if (!StringUtils.isEmpty(sinkSpec.getBuiltin())) {
Connector connector = connectorsManager.getConnector(sinkSpec.getBuiltin());
File archive = connector.getArchivePath().toFile();
String sinkClass = connector.getConnectorDefinition().getSinkClass();
SinkSpec.Builder builder = SinkSpec.newBuilder(functionDetails.getSink());
builder.setClassName(sinkClass);
functionDetails.setSink(builder);
fillSinkTypeClass(functionDetails, connector.getClassLoader(), sinkClass);
return archive;
}
}
if (!StringUtils.isEmpty(functionDetails.getBuiltin())) {
return functionsManager.getFunctionArchive(functionDetails.getBuiltin()).toFile();
}
throw new IOException("Could not find built in archive definition");
}
Aggregations