Search in sources :

Example 1 with FunctionConfig

use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.

the class PulsarFunctionLocalRunTest method testE2EPulsarFunctionLocalRun.

/**
 * Validates pulsar sink e2e functionality on functions.
 *
 * @throws Exception
 */
private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setJar(jarFilePathUrl);
    functionConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).metricsPortStart(metricsPort).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
                for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
                    result = consumerStats.getAvailablePermits() == 1000 && consumerStats.getMetadata() != null && consumerStats.getMetadata().containsKey("id") && consumerStats.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, functionName));
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150));
    // validate pulsar sink consumer has started on the topic
    TopicStats stats = admin.topics().getStats(sourceTopic);
    assertTrue(stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    int totalMsgs = 5;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedPropertyValue = msg.getProperty(propertyKey);
        assertEquals(propertyValue, receivedPropertyValue);
        assertEquals(msg.getValue(), "my-message-" + i + "!");
    }
    // validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked messages
    // due to publish failure
    assertNotEquals(admin.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_function_processed_successfully_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    double totalMsgRecv = 0.0;
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), functionName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
        totalMsgRecv += m.value;
    }
    Assert.assertEquals(totalMsgRecv, totalMsgs);
    // stop functions
    localRunner.stop();
    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);
            return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 20, 150);
    TopicStats topicStats = admin.topics().getStats(sourceTopic);
    assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
    retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
        } catch (PulsarAdminException e) {
            if (e.getStatusCode() == 404) {
                return true;
            }
            return false;
        }
    }, 10, 150);
    try {
        assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
    } catch (PulsarAdminException e) {
        if (e.getStatusCode() != 404) {
            fail();
        }
    }
}
Also used : FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) Arrays(java.util.Arrays) URL(java.net.URL) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Cleanup(lombok.Cleanup) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) Sink(org.apache.pulsar.io.core.Sink) AfterMethod(org.testng.annotations.AfterMethod) ByteBuffer(java.nio.ByteBuffer) URLClassLoader(java.net.URLClassLoader) NarClassLoader(org.apache.pulsar.common.nar.NarClassLoader) Map(java.util.Map) MockedPulsarServiceBaseTest.retryStrategically(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) Assert.assertFalse(org.testng.Assert.assertFalse) Method(java.lang.reflect.Method) Record(org.apache.pulsar.functions.api.Record) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) BeforeClass(org.testng.annotations.BeforeClass) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) Consumer(org.apache.pulsar.client.api.Consumer) FutureUtil(org.apache.pulsar.common.util.FutureUtil) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) LocalBookkeeperEnsemble(org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble) ServiceConfigurationUtils(org.apache.pulsar.broker.ServiceConfigurationUtils) AuthenticationProviderTls(org.apache.pulsar.broker.authentication.AuthenticationProviderTls) PulsarAuthorizationProvider(org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Utils(org.apache.pulsar.common.functions.Utils) DataProvider(org.testng.annotations.DataProvider) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) HashSet(java.util.HashSet) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) Lists(com.google.common.collect.Lists) NarClassLoaderBuilder(org.apache.pulsar.common.nar.NarClassLoaderBuilder) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Assert(org.testng.Assert) SchemaDefinition(org.apache.pulsar.client.api.schema.SchemaDefinition) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TenantInfo(org.apache.pulsar.common.policies.data.TenantInfo) AfterClass(org.testng.annotations.AfterClass) SourceConfig(org.apache.pulsar.common.io.SourceConfig) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) JAVA_INSTANCE_JAR_PROPERTY(org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Assert.fail(org.testng.Assert.fail) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) SinkConfig(org.apache.pulsar.common.io.SinkConfig) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) File(java.io.File) Schema(org.apache.pulsar.client.api.Schema) TimeUnit(java.util.concurrent.TimeUnit) GenericRecord(org.apache.pulsar.client.api.schema.GenericRecord) FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon) SinkContext(org.apache.pulsar.io.core.SinkContext) Authentication(org.apache.pulsar.client.api.Authentication) Assert.assertTrue(org.testng.Assert.assertTrue) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) BrokerStats(org.apache.pulsar.client.admin.BrokerStats) Collections(java.util.Collections) LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) HashMap(java.util.HashMap) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) Cleanup(lombok.Cleanup) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) TopicStats(org.apache.pulsar.common.policies.data.TopicStats)

Example 2 with FunctionConfig

use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.

the class PulsarFunctionPublishTest method testMultipleAddress.

@Test
public void testMultipleAddress() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String publishTopic = "persistent://" + replNamespace + "/publishtopic";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, publishTopic, subscriptionName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    String secondAddress = pulsar.getWebServiceAddressTls().replace("https://", "");
    // set multi webService url
    PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddressTls() + "," + secondAddress).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(authTls).build();
    File jarFile = getPulsarApiExamplesJar();
    Assert.assertTrue(jarFile.exists() && jarFile.isFile());
    pulsarAdmin.functions().createFunction(functionConfig, jarFile.getAbsolutePath());
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
    admin.functions().deleteFunction(tenant, namespacePortion, functionName);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
}
Also used : FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) HashMap(java.util.HashMap) Authentication(org.apache.pulsar.client.api.Authentication) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) File(java.io.File) Test(org.testng.annotations.Test)

Example 3 with FunctionConfig

use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.

the class PulsarFunctionE2ESecurityTest method testAuthorization.

@Test
public void testAuthorization() throws Exception {
    String token1 = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty());
    String token2 = AuthTokenUtils.createToken(secretKey, "wrong-subject", Optional.empty());
    final String replNamespace = TENANT + "/" + NAMESPACE;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    final String subscriptionName = "test-sub";
    // create user admin client
    AuthenticationToken authToken1 = new AuthenticationToken();
    authToken1.configure("token:" + token1);
    AuthenticationToken authToken2 = new AuthenticationToken();
    authToken2.configure("token:" + token2);
    try (PulsarAdmin admin1 = spy(PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).authentication(authToken1).build());
        PulsarAdmin admin2 = spy(PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).authentication(authToken2).build())) {
        String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
        FunctionConfig functionConfig = createFunctionConfig(TENANT, NAMESPACE, functionName, sourceTopic, sinkTopic, subscriptionName);
        // creating function should fail since admin1 doesn't have permissions granted yet
        try {
            admin1.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
            fail("client admin shouldn't have permissions to create function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        // grant permissions to admin1
        Set<AuthAction> actions = new HashSet<>();
        actions.add(AuthAction.functions);
        actions.add(AuthAction.produce);
        actions.add(AuthAction.consume);
        superUserAdmin.namespaces().grantPermissionOnNamespace(replNamespace, SUBJECT, actions);
        // user should be able to create function now
        admin1.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
        // admin2 should still fail
        try {
            admin2.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
            fail("client admin shouldn't have permissions to create function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        // creating on another tenant should also fail
        try {
            admin2.functions().createFunctionWithUrl(createFunctionConfig(TENANT2, NAMESPACE, functionName, sourceTopic, sinkTopic, subscriptionName), jarFilePathUrl);
            fail("client admin shouldn't have permissions to create function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        assertTrue(retryStrategically((test) -> {
            try {
                return admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName).getNumRunning() == 1 && admin1.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
            } catch (PulsarAdminException e) {
                return false;
            }
        }, 50, 150));
        // validate pulsar sink consumer has started on the topic
        assertEquals(admin1.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
        // create a producer that creates a topic at broker
        try (Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
            Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe()) {
            int totalMsgs = 5;
            for (int i = 0; i < totalMsgs; i++) {
                String data = "my-message-" + i;
                producer.newMessage().property(propertyKey, propertyValue).value(data).send();
            }
            retryStrategically((test) -> {
                try {
                    SubscriptionStats subStats = admin1.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
                    return subStats.getUnackedMessages() == 0;
                } catch (PulsarAdminException e) {
                    return false;
                }
            }, 50, 150);
            Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
            String receivedPropertyValue = msg.getProperty(propertyKey);
            assertEquals(propertyValue, receivedPropertyValue);
            // validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked
            // messages
            // due to publish failure
            assertNotEquals(admin1.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
        }
        // test update functions
        functionConfig.setParallelism(2);
        // admin2 should still fail
        try {
            admin2.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
            fail("client admin shouldn't have permissions to update function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
        assertTrue(retryStrategically((test) -> {
            try {
                return admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName).getNumRunning() == 2;
            } catch (PulsarAdminException e) {
                return false;
            }
        }, 50, 150));
        // test getFunctionInfo
        try {
            admin2.functions().getFunction(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to get function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().getFunction(TENANT, NAMESPACE, functionName);
        // test getFunctionInstanceStatus
        try {
            admin2.functions().getFunctionStatus(TENANT, NAMESPACE, functionName, 0);
            fail("client admin shouldn't have permissions to get function status");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName, 0);
        // test getFunctionStatus
        try {
            admin2.functions().getFunctionStatus(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to get function status");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName);
        // test getFunctionStats
        try {
            admin2.functions().getFunctionStats(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to get function stats");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().getFunctionStats(TENANT, NAMESPACE, functionName);
        // test getFunctionInstanceStats
        try {
            admin2.functions().getFunctionStats(TENANT, NAMESPACE, functionName, 0);
            fail("client admin shouldn't have permissions to get function stats");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().getFunctionStats(TENANT, NAMESPACE, functionName, 0);
        // test listFunctions
        try {
            admin2.functions().getFunctions(TENANT, NAMESPACE);
            fail("client admin shouldn't have permissions to list functions");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().getFunctions(TENANT, NAMESPACE);
        // test triggerFunction
        try {
            admin2.functions().triggerFunction(TENANT, NAMESPACE, functionName, sourceTopic, "foo", null);
            fail("client admin shouldn't have permissions to trigger function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().triggerFunction(TENANT, NAMESPACE, functionName, sourceTopic, "foo", null);
        // test restartFunctionInstance
        try {
            admin2.functions().restartFunction(TENANT, NAMESPACE, functionName, 0);
            fail("client admin shouldn't have permissions to restart function instance");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().restartFunction(TENANT, NAMESPACE, functionName, 0);
        // test restartFunctionInstances
        try {
            admin2.functions().restartFunction(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to restart function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().restartFunction(TENANT, NAMESPACE, functionName);
        // test stopFunction instance
        try {
            admin2.functions().stopFunction(TENANT, NAMESPACE, functionName, 0);
            fail("client admin shouldn't have permissions to stop function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().stopFunction(TENANT, NAMESPACE, functionName, 0);
        // test stopFunction all instance
        try {
            admin2.functions().stopFunction(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to restart function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().stopFunction(TENANT, NAMESPACE, functionName);
        // test startFunction instance
        try {
            admin2.functions().startFunction(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to restart function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().restartFunction(TENANT, NAMESPACE, functionName);
        // test startFunction all instances
        try {
            admin2.functions().restartFunction(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to restart function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        admin1.functions().restartFunction(TENANT, NAMESPACE, functionName);
        // admin2 should still fail
        try {
            admin2.functions().deleteFunction(TENANT, NAMESPACE, functionName);
            fail("client admin shouldn't have permissions to delete function");
        } catch (PulsarAdminException.NotAuthorizedException e) {
        }
        try {
            admin1.functions().deleteFunction(TENANT, NAMESPACE, functionName);
        } catch (PulsarAdminException e) {
            // This happens because the request becomes outdated. Lets retry again
            admin1.functions().deleteFunction(TENANT, NAMESPACE, functionName);
        }
        assertTrue(retryStrategically((test) -> {
            try {
                TopicStats stats = admin1.topics().getStats(sourceTopic);
                boolean done = stats.getSubscriptions().size() == 0;
                if (!done) {
                    log.info("Topic subscription is not cleaned up yet : {}", stats);
                }
                return done;
            } catch (PulsarAdminException e) {
                return false;
            }
        }, 50, 150));
    }
}
Also used : FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) URL(java.net.URL) AuthTokenUtils(org.apache.pulsar.broker.authentication.utils.AuthTokenUtils) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) AfterMethod(org.testng.annotations.AfterMethod) AuthAction(org.apache.pulsar.common.policies.data.AuthAction) Map(java.util.Map) MockedPulsarServiceBaseTest.retryStrategically(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically) SignatureAlgorithm(io.jsonwebtoken.SignatureAlgorithm) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) Method(java.lang.reflect.Method) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) Sets(com.google.common.collect.Sets) Consumer(org.apache.pulsar.client.api.Consumer) FutureUtil(org.apache.pulsar.common.util.FutureUtil) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) LocalBookkeeperEnsemble(org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble) AuthenticationToken(org.apache.pulsar.client.impl.auth.AuthenticationToken) ServiceConfigurationUtils(org.apache.pulsar.broker.ServiceConfigurationUtils) PulsarAuthorizationProvider(org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) PulsarFunctionE2ETest(org.apache.pulsar.io.PulsarFunctionE2ETest) SecretKey(javax.crypto.SecretKey) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) DataProvider(org.testng.annotations.DataProvider) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) Assert.assertEquals(org.testng.Assert.assertEquals) PulsarFunctionLocalRunTest.getPulsarApiExamplesJar(org.apache.pulsar.functions.worker.PulsarFunctionLocalRunTest.getPulsarApiExamplesJar) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) HashSet(java.util.HashSet) Lists(com.google.common.collect.Lists) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TenantInfo(org.apache.pulsar.common.policies.data.TenantInfo) Properties(java.util.Properties) Logger(org.slf4j.Logger) AuthenticationProviderToken(org.apache.pulsar.broker.authentication.AuthenticationProviderToken) JAVA_INSTANCE_JAR_PROPERTY(org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Assert.fail(org.testng.Assert.fail) PulsarService(org.apache.pulsar.broker.PulsarService) Schema(org.apache.pulsar.client.api.Schema) TimeUnit(java.util.concurrent.TimeUnit) Assert.assertTrue(org.testng.Assert.assertTrue) BrokerStats(org.apache.pulsar.client.admin.BrokerStats) Collections(java.util.Collections) AuthenticationToken(org.apache.pulsar.client.impl.auth.AuthenticationToken) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) AuthAction(org.apache.pulsar.common.policies.data.AuthAction) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) HashSet(java.util.HashSet) Test(org.testng.annotations.Test) PulsarFunctionE2ETest(org.apache.pulsar.io.PulsarFunctionE2ETest)

Example 4 with FunctionConfig

use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.

the class PulsarFunctionE2ESecurityTest method createFunctionConfig.

protected static FunctionConfig createFunctionConfig(String tenant, String namespace, String functionName, String sourceTopic, String sinkTopic, String subscriptionName) {
    FunctionConfig functionConfig = new FunctionConfig();
    functionConfig.setTenant(tenant);
    functionConfig.setNamespace(namespace);
    functionConfig.setName(functionName);
    functionConfig.setParallelism(1);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE);
    functionConfig.setSubName(subscriptionName);
    functionConfig.setInputs(Collections.singleton(sourceTopic));
    functionConfig.setAutoAck(true);
    functionConfig.setClassName("org.apache.pulsar.functions.api.examples.ExclamationFunction");
    functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
    functionConfig.setOutput(sinkTopic);
    functionConfig.setCleanupSubscription(true);
    return functionConfig;
}
Also used : FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig)

Example 5 with FunctionConfig

use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.

the class PulsarFunctionPublishTest method testUpdateFunctionUserConfig.

@Test
public void testUpdateFunctionUserConfig() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String publishTopic = "persistent://" + replNamespace + "/publishtopic";
    final String functionName = "test-update-user-config";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, publishTopic, subscriptionName);
    String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
    admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
    Map<String, Object> userConfig = functionConfig.getUserConfig();
    functionConfig.setUserConfig(null);
    admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
    FunctionConfig updatefunctionConfig1 = admin.functions().getFunction(tenant, namespacePortion, functionName);
    Assert.assertEquals(userConfig, updatefunctionConfig1.getUserConfig());
    Map<String, Object> newUserConfig = new HashMap<>();
    newUserConfig.put("publish-topic", publishTopic);
    newUserConfig.put("test", "test");
    updatefunctionConfig1.setUserConfig(newUserConfig);
    admin.functions().updateFunctionWithUrl(updatefunctionConfig1, jarFilePathUrl);
    FunctionConfig updatefunctionConfig2 = admin.functions().getFunction(tenant, namespacePortion, functionName);
    Assert.assertEquals(updatefunctionConfig2.getUserConfig(), updatefunctionConfig1.getUserConfig());
}
Also used : FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) HashMap(java.util.HashMap) Test(org.testng.annotations.Test)

Aggregations

FunctionConfig (org.apache.pulsar.common.functions.FunctionConfig)267 Test (org.testng.annotations.Test)177 Gson (com.google.gson.Gson)60 PulsarAdminException (org.apache.pulsar.client.admin.PulsarAdminException)60 HashMap (java.util.HashMap)57 File (java.io.File)48 ConsumerConfig (org.apache.pulsar.common.functions.ConsumerConfig)36 SubscriptionStats (org.apache.pulsar.common.policies.data.SubscriptionStats)36 URL (java.net.URL)27 IOException (java.io.IOException)24 IdentityFunction (org.apache.pulsar.functions.api.utils.IdentityFunction)24 InvalidProtocolBufferException (com.google.protobuf.InvalidProtocolBufferException)21 TopicStats (org.apache.pulsar.common.policies.data.TopicStats)21 Function (org.apache.pulsar.functions.proto.Function)21 FunctionDetails (org.apache.pulsar.functions.proto.Function.FunctionDetails)21 Map (java.util.Map)18 Set (java.util.Set)18 FunctionStatsImpl (org.apache.pulsar.common.policies.data.FunctionStatsImpl)18 ObjectMapperFactory (org.apache.pulsar.common.util.ObjectMapperFactory)18 PulsarAdmin (org.apache.pulsar.client.admin.PulsarAdmin)15