Search in sources :

Example 1 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class PulsarFunctionLocalRunTest method testExitOnError.

@Test(timeOut = 20000)
public void testExitOnError() throws Throwable {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    sinkConfig.setClassName(TestErrorSink.class.getName());
    int metricsPort = FunctionCommon.findAvailablePort();
    LocalRunner.LocalRunnerBuilder localRunnerBuilder = LocalRunner.builder().clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).exitOnError(true);
    sinkConfig.getConfigs().put("throwErrorOpen", true);
    localRunnerBuilder.sinkConfig(sinkConfig);
    LocalRunner localRunner = localRunnerBuilder.build();
    localRunner.start(true);
    sinkConfig.getConfigs().put("throwErrorWrite", true);
    localRunnerBuilder.sinkConfig(sinkConfig);
    localRunner = localRunnerBuilder.build();
    localRunner.start(true);
}
Also used : LocalRunner(org.apache.pulsar.functions.LocalRunner) SinkConfig(org.apache.pulsar.common.io.SinkConfig) Test(org.testng.annotations.Test)

Example 2 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class PulsarFunctionLocalRunTest method createSinkConfig.

private static SinkConfig createSinkConfig(String tenant, String namespace, String functionName, String sourceTopic, String subName) {
    SinkConfig sinkConfig = new SinkConfig();
    sinkConfig.setTenant(tenant);
    sinkConfig.setNamespace(namespace);
    sinkConfig.setName(functionName);
    sinkConfig.setParallelism(1);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().build()));
    sinkConfig.setSourceSubscriptionName(subName);
    sinkConfig.setCleanupSubscription(true);
    sinkConfig.setConfigs(new HashMap<>());
    return sinkConfig;
}
Also used : SinkConfig(org.apache.pulsar.common.io.SinkConfig)

Example 3 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class PulsarSinkE2ETest method testPulsarSinkDLQ.

@Test(timeOut = 30000)
public void testPulsarSinkDLQ() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String dlqTopic = sourceTopic + "-DLQ";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // 1 create producer、DLQ consumer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();
    // 2 setup sink
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setMaxMessageRetries(2);
    sinkConfig.setDeadLetterTopic(dlqTopic);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    sinkConfig.setClassName(SinkForTest.class.getName());
    @Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == 1 && topicStats.getSubscriptions().get(subscriptionName).getConsumers().get(0).getAvailablePermits() == 1000;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // 3 send message
    int totalMsgs = 10;
    Set<String> remainingMessagesToReceive = new HashSet<>();
    for (int i = 0; i < totalMsgs; i++) {
        String messageBody = "fail" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(messageBody).send();
        remainingMessagesToReceive.add(messageBody);
    }
    // 4 All messages should enter DLQ
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
        assertNotNull(message);
        remainingMessagesToReceive.remove(message.getValue());
    }
    assertEquals(remainingMessagesToReceive, Collections.emptySet());
    // clean up
    producer.close();
    consumer.close();
}
Also used : LocalRunner(org.apache.pulsar.functions.LocalRunner) Cleanup(lombok.Cleanup) SinkConfig(org.apache.pulsar.common.io.SinkConfig) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 4 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class PulsarSinkE2ETest method createSinkConfig.

private static SinkConfig createSinkConfig(String tenant, String namespace, String functionName, String sourceTopic, String subName) {
    SinkConfig sinkConfig = new SinkConfig();
    sinkConfig.setTenant(tenant);
    sinkConfig.setNamespace(namespace);
    sinkConfig.setName(functionName);
    sinkConfig.setParallelism(1);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().build()));
    sinkConfig.setSourceSubscriptionName(subName);
    sinkConfig.setCleanupSubscription(true);
    return sinkConfig;
}
Also used : SinkConfig(org.apache.pulsar.common.io.SinkConfig)

Example 5 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class SinksImpl method updateSinkAsync.

@Override
public CompletableFuture<Void> updateSinkAsync(SinkConfig sinkConfig, String fileName, UpdateOptions updateOptions) {
    final CompletableFuture<Void> future = new CompletableFuture<>();
    if (!validateSinkName(sinkConfig.getTenant(), sinkConfig.getNamespace(), sinkConfig.getName(), future)) {
        return future;
    }
    try {
        RequestBuilder builder = put(sink.path(sinkConfig.getTenant()).path(sinkConfig.getNamespace()).path(sinkConfig.getName()).getUri().toASCIIString()).addBodyPart(new StringPart("sinkConfig", ObjectMapperFactory.getThreadLocal().writeValueAsString(sinkConfig), MediaType.APPLICATION_JSON));
        UpdateOptionsImpl options = (UpdateOptionsImpl) updateOptions;
        if (options != null) {
            builder.addBodyPart(new StringPart("updateOptions", ObjectMapperFactory.getThreadLocal().writeValueAsString(options), MediaType.APPLICATION_JSON));
        }
        if (fileName != null && !fileName.startsWith("builtin://")) {
            // If the function code is built in, we don't need to submit here
            builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM));
        }
        asyncHttpClient.executeRequest(addAuthHeaders(sink, builder).build()).toCompletableFuture().thenAccept(response -> {
            if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) {
                future.completeExceptionally(getApiException(Response.status(response.getStatusCode()).entity(response.getResponseBody()).build()));
            } else {
                future.complete(null);
            }
        }).exceptionally(throwable -> {
            future.completeExceptionally(getApiException(throwable));
            return null;
        });
    } catch (Exception e) {
        future.completeExceptionally(getApiException(e));
    }
    return future;
}
Also used : AsyncHttpClient(org.asynchttpclient.AsyncHttpClient) ConnectorDefinition(org.apache.pulsar.common.io.ConnectorDefinition) UpdateOptions(org.apache.pulsar.common.functions.UpdateOptions) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) CompletableFuture(java.util.concurrent.CompletableFuture) Sink(org.apache.pulsar.client.admin.Sink) FormDataMultiPart(org.glassfish.jersey.media.multipart.FormDataMultiPart) StringUtils(org.apache.commons.lang3.StringUtils) MediaType(javax.ws.rs.core.MediaType) Gson(com.google.gson.Gson) InvocationCallback(javax.ws.rs.client.InvocationCallback) FormDataBodyPart(org.glassfish.jersey.media.multipart.FormDataBodyPart) RequestBuilder(org.asynchttpclient.RequestBuilder) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) UpdateOptionsImpl(org.apache.pulsar.common.functions.UpdateOptionsImpl) SinkConfig(org.apache.pulsar.common.io.SinkConfig) SinkStatus(org.apache.pulsar.common.policies.data.SinkStatus) Sinks(org.apache.pulsar.client.admin.Sinks) FilePart(org.asynchttpclient.request.body.multipart.FilePart) Entity(javax.ws.rs.client.Entity) File(java.io.File) Dsl.post(org.asynchttpclient.Dsl.post) GenericType(javax.ws.rs.core.GenericType) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Dsl.put(org.asynchttpclient.Dsl.put) Response(javax.ws.rs.core.Response) Authentication(org.apache.pulsar.client.api.Authentication) StringPart(org.asynchttpclient.request.body.multipart.StringPart) WebTarget(javax.ws.rs.client.WebTarget) CompletableFuture(java.util.concurrent.CompletableFuture) RequestBuilder(org.asynchttpclient.RequestBuilder) StringPart(org.asynchttpclient.request.body.multipart.StringPart) UpdateOptionsImpl(org.apache.pulsar.common.functions.UpdateOptionsImpl) File(java.io.File) FilePart(org.asynchttpclient.request.body.multipart.FilePart) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException)

Aggregations

SinkConfig (org.apache.pulsar.common.io.SinkConfig)223 Test (org.testng.annotations.Test)165 Gson (com.google.gson.Gson)42 HashMap (java.util.HashMap)33 Function (org.apache.pulsar.functions.proto.Function)27 File (java.io.File)24 ConsumerConfig (org.apache.pulsar.common.functions.ConsumerConfig)24 IdentityFunction (org.apache.pulsar.functions.api.utils.IdentityFunction)24 NarClassLoader (org.apache.pulsar.common.nar.NarClassLoader)21 PulsarAdminException (org.apache.pulsar.client.admin.PulsarAdminException)19 IOException (java.io.IOException)18 Resources (org.apache.pulsar.common.functions.Resources)15 FileInputStream (java.io.FileInputStream)12 WebTarget (javax.ws.rs.client.WebTarget)12 ObjectMapperFactory (org.apache.pulsar.common.util.ObjectMapperFactory)12 TopicStats (org.apache.pulsar.common.policies.data.TopicStats)10 List (java.util.List)9 Map (java.util.Map)9 CompletableFuture (java.util.concurrent.CompletableFuture)9 Cleanup (lombok.Cleanup)9