Search in sources :

Example 11 with LocalRunner

use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.

the class PulsarFunctionLocalRunTest method testAvroFunctionLocalRun.

private void testAvroFunctionLocalRun(String jarFilePathUrl) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    Schema schema = Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull(true).withJSR310ConversionEnabled(true).withPojo(avroTestObjectClass).build());
    // use AVRO schema
    admin.schemas().createSchema(sourceTopic, schema.getSchemaInfo());
    // please note that in this test the sink topic schema is different from the schema of the source topic
    // produce message to sourceTopic
    Producer<Object> producer = pulsarClient.newProducer(schema).topic(sourceTopic).create();
    // consume message from sinkTopic
    Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic(sinkTopic).subscriptionName("sub").subscribe();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
    // set jsr310ConversionEnabled、alwaysAllowNull
    Map<String, String> schemaInput = new HashMap<>();
    schemaInput.put(sourceTopic, "{\"schemaType\":\"AVRO\",\"schemaProperties\":{\"__jsr310ConversionEnabled\":\"true\",\"__alwaysAllowNull\":\"true\"}}");
    Map<String, String> schemaOutput = new HashMap<>();
    schemaOutput.put(sinkTopic, "{\"schemaType\":\"AVRO\",\"schemaProperties\":{\"__jsr310ConversionEnabled\":\"true\",\"__alwaysAllowNull\":\"true\"}}");
    functionConfig.setCustomSchemaInputs(schemaInput);
    functionConfig.setCustomSchemaOutputs(schemaOutput);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    if (jarFilePathUrl == null) {
        functionConfig.setClassName("org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction");
    } else {
        functionConfig.setJar(jarFilePathUrl);
    }
    @Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    retryStrategically((test) -> {
        try {
            TopicStats stats = admin.topics().getStats(sourceTopic);
            return stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    int totalMsgs = 5;
    Method setBaseValueMethod = avroTestObjectClass.getMethod("setBaseValue", new Class[] { int.class });
    for (int i = 0; i < totalMsgs; i++) {
        Object avroTestObject = avroTestObjectClass.getDeclaredConstructor().newInstance();
        setBaseValueMethod.invoke(avroTestObject, i);
        producer.newMessage().property(propertyKey, propertyValue).value(avroTestObject).send();
    }
    // consume message from sinkTopic
    for (int i = 0; i < totalMsgs; i++) {
        Message<GenericRecord> msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedPropertyValue = msg.getProperty(propertyKey);
        assertEquals(propertyValue, receivedPropertyValue);
        assertEquals(msg.getValue().getField("baseValue"), 10 + i);
        consumer.acknowledge(msg);
    }
    // validate pulsar-sink consumer has consumed all messages
    assertNotEquals(admin.topics().getStats(sinkTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), 0);
    localRunner.stop();
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150);
    // change the schema, the function should not run, resulting in no messages to consume
    schemaInput.put(sourceTopic, "{\"schemaType\":\"AVRO\",\"schemaProperties\":{\"__jsr310ConversionEnabled\":\"false\",\"__alwaysAllowNull\":\"false\"}}");
    localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    producer.newMessage().property(propertyKey, propertyValue).value(avroTestObjectClass.getDeclaredConstructor().newInstance()).send();
    Message<GenericRecord> msg = consumer.receive(2, TimeUnit.SECONDS);
    Assert.assertNull(msg);
    producer.close();
    consumer.close();
    localRunner.stop();
}
Also used : FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) LocalRunner(org.apache.pulsar.functions.LocalRunner) HashMap(java.util.HashMap) Schema(org.apache.pulsar.client.api.Schema) AfterMethod(org.testng.annotations.AfterMethod) Method(java.lang.reflect.Method) BeforeMethod(org.testng.annotations.BeforeMethod) Cleanup(lombok.Cleanup) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) GenericRecord(org.apache.pulsar.client.api.schema.GenericRecord) TopicStats(org.apache.pulsar.common.policies.data.TopicStats)

Example 12 with LocalRunner

use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.

the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.

private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    if (className != null) {
        sinkConfig.setClassName(className);
    } else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
        sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
    }
    sinkConfig.setArchive(jarFilePathUrl);
    sinkConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000;
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150));
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200));
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_sink_written_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        } else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
            if (m == null) {
                m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
            }
            assertEquals(m.value, 0);
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalNumRecvMsg = 0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), sinkName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
        totalNumRecvMsg += m.value;
    }
    assertEquals(totalNumRecvMsg, totalMsgs);
    // stop sink
    localRunner.stop();
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            TopicStats stats = admin.topics().getStats(sourceTopic);
            return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150));
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}
Also used : LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) Arrays(java.util.Arrays) URL(java.net.URL) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Cleanup(lombok.Cleanup) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) Sink(org.apache.pulsar.io.core.Sink) AfterMethod(org.testng.annotations.AfterMethod) ByteBuffer(java.nio.ByteBuffer) URLClassLoader(java.net.URLClassLoader) NarClassLoader(org.apache.pulsar.common.nar.NarClassLoader) Map(java.util.Map) MockedPulsarServiceBaseTest.retryStrategically(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) Assert.assertFalse(org.testng.Assert.assertFalse) Method(java.lang.reflect.Method) Record(org.apache.pulsar.functions.api.Record) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) BeforeClass(org.testng.annotations.BeforeClass) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) Consumer(org.apache.pulsar.client.api.Consumer) FutureUtil(org.apache.pulsar.common.util.FutureUtil) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) LocalBookkeeperEnsemble(org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble) ServiceConfigurationUtils(org.apache.pulsar.broker.ServiceConfigurationUtils) AuthenticationProviderTls(org.apache.pulsar.broker.authentication.AuthenticationProviderTls) PulsarAuthorizationProvider(org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Utils(org.apache.pulsar.common.functions.Utils) DataProvider(org.testng.annotations.DataProvider) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) HashSet(java.util.HashSet) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) Lists(com.google.common.collect.Lists) NarClassLoaderBuilder(org.apache.pulsar.common.nar.NarClassLoaderBuilder) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Assert(org.testng.Assert) SchemaDefinition(org.apache.pulsar.client.api.schema.SchemaDefinition) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TenantInfo(org.apache.pulsar.common.policies.data.TenantInfo) AfterClass(org.testng.annotations.AfterClass) SourceConfig(org.apache.pulsar.common.io.SourceConfig) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) JAVA_INSTANCE_JAR_PROPERTY(org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Assert.fail(org.testng.Assert.fail) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) SinkConfig(org.apache.pulsar.common.io.SinkConfig) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) File(java.io.File) Schema(org.apache.pulsar.client.api.Schema) TimeUnit(java.util.concurrent.TimeUnit) GenericRecord(org.apache.pulsar.client.api.schema.GenericRecord) FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon) SinkContext(org.apache.pulsar.io.core.SinkContext) Authentication(org.apache.pulsar.client.api.Authentication) Assert.assertTrue(org.testng.Assert.assertTrue) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) BrokerStats(org.apache.pulsar.client.admin.BrokerStats) Collections(java.util.Collections) LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) HashMap(java.util.HashMap) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) Cleanup(lombok.Cleanup) SinkConfig(org.apache.pulsar.common.io.SinkConfig) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Map(java.util.Map) HashMap(java.util.HashMap) TopicStats(org.apache.pulsar.common.policies.data.TopicStats)

Example 13 with LocalRunner

use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.

the class PulsarSinkE2ETest method testPulsarSinkDLQ.

@Test(timeOut = 30000)
public void testPulsarSinkDLQ() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String dlqTopic = sourceTopic + "-DLQ";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // 1 create producer、DLQ consumer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();
    // 2 setup sink
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setMaxMessageRetries(2);
    sinkConfig.setDeadLetterTopic(dlqTopic);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    sinkConfig.setClassName(SinkForTest.class.getName());
    @Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == 1 && topicStats.getSubscriptions().get(subscriptionName).getConsumers().get(0).getAvailablePermits() == 1000;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // 3 send message
    int totalMsgs = 10;
    Set<String> remainingMessagesToReceive = new HashSet<>();
    for (int i = 0; i < totalMsgs; i++) {
        String messageBody = "fail" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(messageBody).send();
        remainingMessagesToReceive.add(messageBody);
    }
    // 4 All messages should enter DLQ
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
        assertNotNull(message);
        remainingMessagesToReceive.remove(message.getValue());
    }
    assertEquals(remainingMessagesToReceive, Collections.emptySet());
    // clean up
    producer.close();
    consumer.close();
}
Also used : LocalRunner(org.apache.pulsar.functions.LocalRunner) Cleanup(lombok.Cleanup) SinkConfig(org.apache.pulsar.common.io.SinkConfig) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 14 with LocalRunner

use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.

the class BatchDataGeneratorExec method main.

public static void main(final String[] args) throws Exception {
    final String cronString = "0 0/5 * * * ?";
    final Map<String, Object> discoveryConfig = new HashMap<>();
    discoveryConfig.put(CronTriggerer.CRON_KEY, cronString);
    final BatchSourceConfig batchSourceConfig = BatchSourceConfig.builder().discoveryTriggererClassName(CronTriggerer.class.getName()).discoveryTriggererConfig(discoveryConfig).build();
    final SourceConfig sourceConfig = SourceConfig.builder().batchSourceConfig(batchSourceConfig).className(BatchDataGeneratorSource.class.getName()).configs(new HashMap<>()).name("BatchDataGenerator").parallelism(1).topicName("persistent://public/default/batchdatagenerator").build();
    final LocalRunner localRunner = LocalRunner.builder().brokerServiceUrl("pulsar://localhost:6650").sourceConfig(sourceConfig).build();
    localRunner.start(false);
    TimeUnit.MINUTES.sleep(30);
    localRunner.stop();
    System.exit(0);
}
Also used : BatchSourceConfig(org.apache.pulsar.common.io.BatchSourceConfig) CronTriggerer(org.apache.pulsar.io.batchdiscovery.CronTriggerer) LocalRunner(org.apache.pulsar.functions.LocalRunner) HashMap(java.util.HashMap) BatchSourceConfig(org.apache.pulsar.common.io.BatchSourceConfig) SourceConfig(org.apache.pulsar.common.io.SourceConfig)

Example 15 with LocalRunner

use of org.apache.pulsar.functions.LocalRunner in project pulsar by yahoo.

the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.

private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    if (className != null) {
        sinkConfig.setClassName(className);
    } else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
        sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
    }
    sinkConfig.setArchive(jarFilePathUrl);
    sinkConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000;
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150));
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200));
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_sink_written_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        } else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
            if (m == null) {
                m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
            }
            assertEquals(m.value, 0);
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalNumRecvMsg = 0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), sinkName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
        totalNumRecvMsg += m.value;
    }
    assertEquals(totalNumRecvMsg, totalMsgs);
    // stop sink
    localRunner.stop();
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            TopicStats stats = admin.topics().getStats(sourceTopic);
            return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150));
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}
Also used : LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) Arrays(java.util.Arrays) URL(java.net.URL) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Cleanup(lombok.Cleanup) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) Sink(org.apache.pulsar.io.core.Sink) AfterMethod(org.testng.annotations.AfterMethod) ByteBuffer(java.nio.ByteBuffer) URLClassLoader(java.net.URLClassLoader) NarClassLoader(org.apache.pulsar.common.nar.NarClassLoader) Map(java.util.Map) MockedPulsarServiceBaseTest.retryStrategically(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) Assert.assertFalse(org.testng.Assert.assertFalse) Method(java.lang.reflect.Method) Record(org.apache.pulsar.functions.api.Record) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) BeforeClass(org.testng.annotations.BeforeClass) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) Consumer(org.apache.pulsar.client.api.Consumer) FutureUtil(org.apache.pulsar.common.util.FutureUtil) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) LocalBookkeeperEnsemble(org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble) ServiceConfigurationUtils(org.apache.pulsar.broker.ServiceConfigurationUtils) AuthenticationProviderTls(org.apache.pulsar.broker.authentication.AuthenticationProviderTls) PulsarAuthorizationProvider(org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Utils(org.apache.pulsar.common.functions.Utils) DataProvider(org.testng.annotations.DataProvider) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) HashSet(java.util.HashSet) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) Lists(com.google.common.collect.Lists) NarClassLoaderBuilder(org.apache.pulsar.common.nar.NarClassLoaderBuilder) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Assert(org.testng.Assert) SchemaDefinition(org.apache.pulsar.client.api.schema.SchemaDefinition) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TenantInfo(org.apache.pulsar.common.policies.data.TenantInfo) AfterClass(org.testng.annotations.AfterClass) SourceConfig(org.apache.pulsar.common.io.SourceConfig) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) JAVA_INSTANCE_JAR_PROPERTY(org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Assert.fail(org.testng.Assert.fail) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) SinkConfig(org.apache.pulsar.common.io.SinkConfig) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) File(java.io.File) Schema(org.apache.pulsar.client.api.Schema) TimeUnit(java.util.concurrent.TimeUnit) GenericRecord(org.apache.pulsar.client.api.schema.GenericRecord) FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon) SinkContext(org.apache.pulsar.io.core.SinkContext) Authentication(org.apache.pulsar.client.api.Authentication) Assert.assertTrue(org.testng.Assert.assertTrue) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) BrokerStats(org.apache.pulsar.client.admin.BrokerStats) Collections(java.util.Collections) LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) HashMap(java.util.HashMap) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) Cleanup(lombok.Cleanup) SinkConfig(org.apache.pulsar.common.io.SinkConfig) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Map(java.util.Map) HashMap(java.util.HashMap) TopicStats(org.apache.pulsar.common.policies.data.TopicStats)

Aggregations

LocalRunner (org.apache.pulsar.functions.LocalRunner)21 HashMap (java.util.HashMap)15 Cleanup (lombok.Cleanup)15 PulsarAdminException (org.apache.pulsar.client.admin.PulsarAdminException)15 SinkConfig (org.apache.pulsar.common.io.SinkConfig)15 TopicStats (org.apache.pulsar.common.policies.data.TopicStats)15 Test (org.testng.annotations.Test)15 Method (java.lang.reflect.Method)12 HashSet (java.util.HashSet)12 Schema (org.apache.pulsar.client.api.Schema)12 GenericRecord (org.apache.pulsar.client.api.schema.GenericRecord)12 AuthenticationTls (org.apache.pulsar.client.impl.auth.AuthenticationTls)12 FunctionConfig (org.apache.pulsar.common.functions.FunctionConfig)12 SourceConfig (org.apache.pulsar.common.io.SourceConfig)12 AfterMethod (org.testng.annotations.AfterMethod)12 BeforeMethod (org.testng.annotations.BeforeMethod)12 Lists (com.google.common.collect.Lists)9 Sets (com.google.common.collect.Sets)9 File (java.io.File)9 IOException (java.io.IOException)9