use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.
the class PulsarFunctionLocalRunTest method testExitOnError.
@Test(timeOut = 20000)
public void testExitOnError() throws Throwable {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// create a producer that creates a topic at broker
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
sinkConfig.setClassName(TestErrorSink.class.getName());
int metricsPort = FunctionCommon.findAvailablePort();
LocalRunner.LocalRunnerBuilder localRunnerBuilder = LocalRunner.builder().clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).exitOnError(true);
sinkConfig.getConfigs().put("throwErrorOpen", true);
localRunnerBuilder.sinkConfig(sinkConfig);
LocalRunner localRunner = localRunnerBuilder.build();
localRunner.start(true);
sinkConfig.getConfigs().put("throwErrorWrite", true);
localRunnerBuilder.sinkConfig(sinkConfig);
localRunner = localRunnerBuilder.build();
localRunner.start(true);
}
use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.
the class PulsarFunctionLocalRunTest method createSinkConfig.
private static SinkConfig createSinkConfig(String tenant, String namespace, String functionName, String sourceTopic, String subName) {
SinkConfig sinkConfig = new SinkConfig();
sinkConfig.setTenant(tenant);
sinkConfig.setNamespace(namespace);
sinkConfig.setName(functionName);
sinkConfig.setParallelism(1);
sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().build()));
sinkConfig.setSourceSubscriptionName(subName);
sinkConfig.setCleanupSubscription(true);
sinkConfig.setConfigs(new HashMap<>());
return sinkConfig;
}
use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.
the class PulsarSinkE2ETest method testPulsarSinkDLQ.
@Test(timeOut = 30000)
public void testPulsarSinkDLQ() throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String dlqTopic = sourceTopic + "-DLQ";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// 1 create producer、DLQ consumer
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();
// 2 setup sink
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
sinkConfig.setMaxMessageRetries(2);
sinkConfig.setDeadLetterTopic(dlqTopic);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
sinkConfig.setClassName(SinkForTest.class.getName());
@Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
localRunner.start(false);
retryStrategically((test) -> {
try {
TopicStats topicStats = admin.topics().getStats(sourceTopic);
return topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == 1 && topicStats.getSubscriptions().get(subscriptionName).getConsumers().get(0).getAvailablePermits() == 1000;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
// 3 send message
int totalMsgs = 10;
Set<String> remainingMessagesToReceive = new HashSet<>();
for (int i = 0; i < totalMsgs; i++) {
String messageBody = "fail" + i;
producer.newMessage().property(propertyKey, propertyValue).value(messageBody).send();
remainingMessagesToReceive.add(messageBody);
}
// 4 All messages should enter DLQ
for (int i = 0; i < totalMsgs; i++) {
Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
assertNotNull(message);
remainingMessagesToReceive.remove(message.getValue());
}
assertEquals(remainingMessagesToReceive, Collections.emptySet());
// clean up
producer.close();
consumer.close();
}
use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.
the class PulsarSinkE2ETest method createSinkConfig.
private static SinkConfig createSinkConfig(String tenant, String namespace, String functionName, String sourceTopic, String subName) {
SinkConfig sinkConfig = new SinkConfig();
sinkConfig.setTenant(tenant);
sinkConfig.setNamespace(namespace);
sinkConfig.setName(functionName);
sinkConfig.setParallelism(1);
sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().build()));
sinkConfig.setSourceSubscriptionName(subName);
sinkConfig.setCleanupSubscription(true);
return sinkConfig;
}
use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.
the class SinksImpl method updateSinkAsync.
@Override
public CompletableFuture<Void> updateSinkAsync(SinkConfig sinkConfig, String fileName, UpdateOptions updateOptions) {
final CompletableFuture<Void> future = new CompletableFuture<>();
if (!validateSinkName(sinkConfig.getTenant(), sinkConfig.getNamespace(), sinkConfig.getName(), future)) {
return future;
}
try {
RequestBuilder builder = put(sink.path(sinkConfig.getTenant()).path(sinkConfig.getNamespace()).path(sinkConfig.getName()).getUri().toASCIIString()).addBodyPart(new StringPart("sinkConfig", ObjectMapperFactory.getThreadLocal().writeValueAsString(sinkConfig), MediaType.APPLICATION_JSON));
UpdateOptionsImpl options = (UpdateOptionsImpl) updateOptions;
if (options != null) {
builder.addBodyPart(new StringPart("updateOptions", ObjectMapperFactory.getThreadLocal().writeValueAsString(options), MediaType.APPLICATION_JSON));
}
if (fileName != null && !fileName.startsWith("builtin://")) {
// If the function code is built in, we don't need to submit here
builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM));
}
asyncHttpClient.executeRequest(addAuthHeaders(sink, builder).build()).toCompletableFuture().thenAccept(response -> {
if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) {
future.completeExceptionally(getApiException(Response.status(response.getStatusCode()).entity(response.getResponseBody()).build()));
} else {
future.complete(null);
}
}).exceptionally(throwable -> {
future.completeExceptionally(getApiException(throwable));
return null;
});
} catch (Exception e) {
future.completeExceptionally(getApiException(e));
}
return future;
}
Aggregations