Examples with Sink org.apache.pulsar.io.core.Sink used on opensource projects

Example 1 with Sink

use of org.apache.pulsar.io.core.Sink in project pulsar by apache.

the class PulsarFunctionLocalRunTest method testE2EPulsarFunctionLocalRun.

/**
 * Validates pulsar sink e2e functionality on functions.
 *
 * @throws Exception
 */
private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setJar(jarFilePathUrl);
    functionConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).metricsPortStart(metricsPort).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000 && consumerStats.getMetadata() != null && consumerStats.getMetadata().containsKey("id") && consumerStats.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, functionName));
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150));
    // validate pulsar sink consumer has started on the topic
    TopicStats stats = admin.topics().getStats(sourceTopic);
    assertTrue(stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    int totalMsgs = 5;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedPropertyValue = msg.getProperty(propertyKey);
        assertEquals(propertyValue, receivedPropertyValue);
        assertEquals(msg.getValue(), "my-message-" + i + "!");
    }
    // validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked messages
    // due to publish failure
    assertNotEquals(admin.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_function_processed_successfully_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalMsgRecv = 0.0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), functionName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
        totalMsgRecv += m.value;
    }
    Assert.assertEquals(totalMsgRecv, totalMsgs);
    // stop functions
    localRunner.stop();
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150);
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
        } catch (PulsarAdminException e) {
            if (e.getStatusCode() == 404) {
                return true;
            }
            return false;
        }
    }, 10, 150);
    try {
        assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
    } catch (PulsarAdminException e) {
        if (e.getStatusCode() != 404) {
            fail();
        }
    }
}

Example 2 with Sink

use of org.apache.pulsar.io.core.Sink in project pulsar by yahoo.

the class PulsarFunctionLocalRunTest method testE2EPulsarFunctionLocalRun.

/**
 * Validates pulsar sink e2e functionality on functions.
 *
 * @throws Exception
 */
private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setJar(jarFilePathUrl);
    functionConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).metricsPortStart(metricsPort).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000 && consumerStats.getMetadata() != null && consumerStats.getMetadata().containsKey("id") && consumerStats.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, functionName));
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150));
    // validate pulsar sink consumer has started on the topic
    TopicStats stats = admin.topics().getStats(sourceTopic);
    assertTrue(stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    int totalMsgs = 5;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedPropertyValue = msg.getProperty(propertyKey);
        assertEquals(propertyValue, receivedPropertyValue);
        assertEquals(msg.getValue(), "my-message-" + i + "!");
    }
    // validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked messages
    // due to publish failure
    assertNotEquals(admin.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_function_processed_successfully_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalMsgRecv = 0.0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), functionName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
        totalMsgRecv += m.value;
    }
    Assert.assertEquals(totalMsgRecv, totalMsgs);
    // stop functions
    localRunner.stop();
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150);
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
        } catch (PulsarAdminException e) {
            if (e.getStatusCode() == 404) {
                return true;
            }
            return false;
        }
    }, 10, 150);
    try {
        assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
    } catch (PulsarAdminException e) {
        if (e.getStatusCode() != 404) {
            fail();
        }
    }
}

Example 3 with Sink

use of org.apache.pulsar.io.core.Sink in project pulsar by yahoo.

the class JavaInstanceRunnable method setupOutput.

private void setupOutput(ContextImpl contextImpl) throws Exception {
    SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
    Object object;
    // If sink classname is not set, we default pulsar sink
    if (sinkSpec.getClassName().isEmpty()) {
        if (StringUtils.isEmpty(sinkSpec.getTopic())) {
            object = PulsarSinkDisable.INSTANCE;
        } else {
            PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
            pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
            pulsarSinkConfig.setTopic(sinkSpec.getTopic());
            pulsarSinkConfig.setForwardSourceMessageProperty(this.instanceConfig.getFunctionDetails().getSink().getForwardSourceMessageProperty());
            if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
                pulsarSinkConfig.setSchemaType(sinkSpec.getSchemaType());
            } else if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
                pulsarSinkConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
            }
            pulsarSinkConfig.setTypeClassName(sinkSpec.getTypeClassName());
            pulsarSinkConfig.setSchemaProperties(sinkSpec.getSchemaPropertiesMap());
            if (this.instanceConfig.getFunctionDetails().getSink().getProducerSpec() != null) {
                org.apache.pulsar.functions.proto.Function.ProducerSpec conf = this.instanceConfig.getFunctionDetails().getSink().getProducerSpec();
                ProducerConfig.ProducerConfigBuilder builder = ProducerConfig.builder().maxPendingMessages(conf.getMaxPendingMessages()).maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()).batchBuilder(conf.getBatchBuilder()).useThreadLocalProducers(conf.getUseThreadLocalProducers()).cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
                pulsarSinkConfig.setProducerConfig(builder.build());
            }
            object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, this.functionClassLoader);
        }
    } else {
        object = Reflections.createInstance(sinkSpec.getClassName(), this.functionClassLoader);
    }
    if (object instanceof Sink) {
        this.sink = (Sink) object;
    } else {
        throw new RuntimeException("Sink does not implement correct interface");
    }
    if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
        Thread.currentThread().setContextClassLoader(this.functionClassLoader);
    }
    try {
        if (sinkSpec.getConfigs().isEmpty()) {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString());
            }
            this.sink.open(new HashMap<>(), contextImpl);
        } else {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec, contextImpl.toString());
            }
            this.sink.open(ObjectMapperFactory.getThreadLocal().readValue(sinkSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
            }), contextImpl);
        }
    } catch (Exception e) {
        log.error("Sink open produced uncaught exception: ", e);
        throw e;
    } finally {
        Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
    }
}

Example 4 with Sink

use of org.apache.pulsar.io.core.Sink in project incubator-pulsar by apache.

the class PulsarFunctionLocalRunTest method testE2EPulsarFunctionLocalRun.

/**
 * Validates pulsar sink e2e functionality on functions.
 *
 * @throws Exception
 */
private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setJar(jarFilePathUrl);
    functionConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).metricsPortStart(metricsPort).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000 && consumerStats.getMetadata() != null && consumerStats.getMetadata().containsKey("id") && consumerStats.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, functionName));
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150));
    // validate pulsar sink consumer has started on the topic
    TopicStats stats = admin.topics().getStats(sourceTopic);
    assertTrue(stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    int totalMsgs = 5;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedPropertyValue = msg.getProperty(propertyKey);
        assertEquals(propertyValue, receivedPropertyValue);
        assertEquals(msg.getValue(), "my-message-" + i + "!");
    }
    // validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked messages
    // due to publish failure
    assertNotEquals(admin.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_function_processed_successfully_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalMsgRecv = 0.0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), functionName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
        totalMsgRecv += m.value;
    }
    Assert.assertEquals(totalMsgRecv, totalMsgs);
    // stop functions
    localRunner.stop();
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150);
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
        } catch (PulsarAdminException e) {
            if (e.getStatusCode() == 404) {
                return true;
            }
            return false;
        }
    }, 10, 150);
    try {
        assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
    } catch (PulsarAdminException e) {
        if (e.getStatusCode() != 404) {
            fail();
        }
    }
}

Example 5 with Sink

use of org.apache.pulsar.io.core.Sink in project incubator-pulsar by apache.

the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.

private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    if (className != null) {
        sinkConfig.setClassName(className);
    } else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
        sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
    }
    sinkConfig.setArchive(jarFilePathUrl);
    sinkConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000;
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150));
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200));
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_sink_written_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        } else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
            if (m == null) {
                m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
            }
            assertEquals(m.value, 0);
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalNumRecvMsg = 0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), sinkName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
        totalNumRecvMsg += m.value;
    }
    assertEquals(totalNumRecvMsg, totalMsgs);
    // stop sink
    localRunner.stop();
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            TopicStats stats = admin.topics().getStats(sourceTopic);
            return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150));
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}