use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.
the class PulsarFunctionLocalRunTest method testE2EPulsarFunctionLocalRun.
/**
* Validates pulsar sink e2e functionality on functions.
*
* @throws Exception
*/
private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
final String sinkTopic = "persistent://" + replNamespace + "/output";
final String propertyKey = "key";
final String propertyValue = "value";
final String functionName = "PulsarFunction-test";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// create a producer that creates a topic at broker
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe();
FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
functionConfig.setJar(jarFilePathUrl);
functionConfig.setParallelism(parallelism);
int metricsPort = FunctionCommon.findAvailablePort();
@Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).metricsPortStart(metricsPort).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
localRunner.start(false);
Assert.assertTrue(retryStrategically((test) -> {
try {
boolean result = false;
TopicStats topicStats = admin.topics().getStats(sourceTopic);
if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
result = consumerStats.getAvailablePermits() == 1000 && consumerStats.getMetadata() != null && consumerStats.getMetadata().containsKey("id") && consumerStats.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, functionName));
}
}
return result;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150));
// validate pulsar sink consumer has started on the topic
TopicStats stats = admin.topics().getStats(sourceTopic);
assertTrue(stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
int totalMsgs = 5;
for (int i = 0; i < totalMsgs; i++) {
String data = "my-message-" + i;
producer.newMessage().property(propertyKey, propertyValue).value(data).send();
}
retryStrategically((test) -> {
try {
SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
return subStats.getUnackedMessages() == 0;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
for (int i = 0; i < totalMsgs; i++) {
Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
String receivedPropertyValue = msg.getProperty(propertyKey);
assertEquals(propertyValue, receivedPropertyValue);
assertEquals(msg.getValue(), "my-message-" + i + "!");
}
// validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked messages
// due to publish failure
assertNotEquals(admin.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
// validate prometheus metrics
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
log.info("prometheus metrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
if (line.startsWith("pulsar_function_processed_successfully_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total");
if (m != null) {
metricsMap.put(m.tags.get("instance_id"), m);
}
}
});
Assert.assertEquals(metricsMap.size(), parallelism);
double totalMsgRecv = 0.0;
for (int i = 0; i < parallelism; i++) {
PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
Assert.assertNotNull(m);
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), String.valueOf(i));
assertEquals(m.tags.get("name"), functionName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
totalMsgRecv += m.value;
}
Assert.assertEquals(totalMsgRecv, totalMsgs);
// stop functions
localRunner.stop();
retryStrategically((test) -> {
try {
TopicStats topicStats = admin.topics().getStats(sourceTopic);
return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150);
TopicStats topicStats = admin.topics().getStats(sourceTopic);
assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
retryStrategically((test) -> {
try {
return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
} catch (PulsarAdminException e) {
if (e.getStatusCode() == 404) {
return true;
}
return false;
}
}, 10, 150);
try {
assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
} catch (PulsarAdminException e) {
if (e.getStatusCode() != 404) {
fail();
}
}
}
use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.
the class PulsarFunctionPublishTest method testMultipleAddress.
@Test
public void testMultipleAddress() throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String publishTopic = "persistent://" + replNamespace + "/publishtopic";
final String functionName = "PulsarFunction-test";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, publishTopic, subscriptionName);
Map<String, String> authParams = new HashMap<>();
authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
Authentication authTls = new AuthenticationTls();
authTls.configure(authParams);
String secondAddress = pulsar.getWebServiceAddressTls().replace("https://", "");
// set multi webService url
PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddressTls() + "," + secondAddress).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(authTls).build();
File jarFile = getPulsarApiExamplesJar();
Assert.assertTrue(jarFile.exists() && jarFile.isFile());
pulsarAdmin.functions().createFunction(functionConfig, jarFile.getAbsolutePath());
retryStrategically((test) -> {
try {
return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
admin.functions().deleteFunction(tenant, namespacePortion, functionName);
retryStrategically((test) -> {
try {
return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 0;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
}
use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.
the class PulsarFunctionE2ESecurityTest method testAuthorization.
@Test
public void testAuthorization() throws Exception {
String token1 = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty());
String token2 = AuthTokenUtils.createToken(secretKey, "wrong-subject", Optional.empty());
final String replNamespace = TENANT + "/" + NAMESPACE;
final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
final String sinkTopic = "persistent://" + replNamespace + "/output";
final String propertyKey = "key";
final String propertyValue = "value";
final String functionName = "PulsarFunction-test";
final String subscriptionName = "test-sub";
// create user admin client
AuthenticationToken authToken1 = new AuthenticationToken();
authToken1.configure("token:" + token1);
AuthenticationToken authToken2 = new AuthenticationToken();
authToken2.configure("token:" + token2);
try (PulsarAdmin admin1 = spy(PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).authentication(authToken1).build());
PulsarAdmin admin2 = spy(PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).authentication(authToken2).build())) {
String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
FunctionConfig functionConfig = createFunctionConfig(TENANT, NAMESPACE, functionName, sourceTopic, sinkTopic, subscriptionName);
// creating function should fail since admin1 doesn't have permissions granted yet
try {
admin1.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
fail("client admin shouldn't have permissions to create function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
// grant permissions to admin1
Set<AuthAction> actions = new HashSet<>();
actions.add(AuthAction.functions);
actions.add(AuthAction.produce);
actions.add(AuthAction.consume);
superUserAdmin.namespaces().grantPermissionOnNamespace(replNamespace, SUBJECT, actions);
// user should be able to create function now
admin1.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
// admin2 should still fail
try {
admin2.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
fail("client admin shouldn't have permissions to create function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
// creating on another tenant should also fail
try {
admin2.functions().createFunctionWithUrl(createFunctionConfig(TENANT2, NAMESPACE, functionName, sourceTopic, sinkTopic, subscriptionName), jarFilePathUrl);
fail("client admin shouldn't have permissions to create function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
assertTrue(retryStrategically((test) -> {
try {
return admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName).getNumRunning() == 1 && admin1.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150));
// validate pulsar sink consumer has started on the topic
assertEquals(admin1.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
// create a producer that creates a topic at broker
try (Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub").subscribe()) {
int totalMsgs = 5;
for (int i = 0; i < totalMsgs; i++) {
String data = "my-message-" + i;
producer.newMessage().property(propertyKey, propertyValue).value(data).send();
}
retryStrategically((test) -> {
try {
SubscriptionStats subStats = admin1.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
return subStats.getUnackedMessages() == 0;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
Message<String> msg = consumer.receive(5, TimeUnit.SECONDS);
String receivedPropertyValue = msg.getProperty(propertyKey);
assertEquals(propertyValue, receivedPropertyValue);
// validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked
// messages
// due to publish failure
assertNotEquals(admin1.topics().getStats(sourceTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), totalMsgs);
}
// test update functions
functionConfig.setParallelism(2);
// admin2 should still fail
try {
admin2.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
fail("client admin shouldn't have permissions to update function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
assertTrue(retryStrategically((test) -> {
try {
return admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName).getNumRunning() == 2;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150));
// test getFunctionInfo
try {
admin2.functions().getFunction(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to get function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().getFunction(TENANT, NAMESPACE, functionName);
// test getFunctionInstanceStatus
try {
admin2.functions().getFunctionStatus(TENANT, NAMESPACE, functionName, 0);
fail("client admin shouldn't have permissions to get function status");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName, 0);
// test getFunctionStatus
try {
admin2.functions().getFunctionStatus(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to get function status");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().getFunctionStatus(TENANT, NAMESPACE, functionName);
// test getFunctionStats
try {
admin2.functions().getFunctionStats(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to get function stats");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().getFunctionStats(TENANT, NAMESPACE, functionName);
// test getFunctionInstanceStats
try {
admin2.functions().getFunctionStats(TENANT, NAMESPACE, functionName, 0);
fail("client admin shouldn't have permissions to get function stats");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().getFunctionStats(TENANT, NAMESPACE, functionName, 0);
// test listFunctions
try {
admin2.functions().getFunctions(TENANT, NAMESPACE);
fail("client admin shouldn't have permissions to list functions");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().getFunctions(TENANT, NAMESPACE);
// test triggerFunction
try {
admin2.functions().triggerFunction(TENANT, NAMESPACE, functionName, sourceTopic, "foo", null);
fail("client admin shouldn't have permissions to trigger function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().triggerFunction(TENANT, NAMESPACE, functionName, sourceTopic, "foo", null);
// test restartFunctionInstance
try {
admin2.functions().restartFunction(TENANT, NAMESPACE, functionName, 0);
fail("client admin shouldn't have permissions to restart function instance");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().restartFunction(TENANT, NAMESPACE, functionName, 0);
// test restartFunctionInstances
try {
admin2.functions().restartFunction(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to restart function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().restartFunction(TENANT, NAMESPACE, functionName);
// test stopFunction instance
try {
admin2.functions().stopFunction(TENANT, NAMESPACE, functionName, 0);
fail("client admin shouldn't have permissions to stop function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().stopFunction(TENANT, NAMESPACE, functionName, 0);
// test stopFunction all instance
try {
admin2.functions().stopFunction(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to restart function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().stopFunction(TENANT, NAMESPACE, functionName);
// test startFunction instance
try {
admin2.functions().startFunction(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to restart function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().restartFunction(TENANT, NAMESPACE, functionName);
// test startFunction all instances
try {
admin2.functions().restartFunction(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to restart function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
admin1.functions().restartFunction(TENANT, NAMESPACE, functionName);
// admin2 should still fail
try {
admin2.functions().deleteFunction(TENANT, NAMESPACE, functionName);
fail("client admin shouldn't have permissions to delete function");
} catch (PulsarAdminException.NotAuthorizedException e) {
}
try {
admin1.functions().deleteFunction(TENANT, NAMESPACE, functionName);
} catch (PulsarAdminException e) {
// This happens because the request becomes outdated. Lets retry again
admin1.functions().deleteFunction(TENANT, NAMESPACE, functionName);
}
assertTrue(retryStrategically((test) -> {
try {
TopicStats stats = admin1.topics().getStats(sourceTopic);
boolean done = stats.getSubscriptions().size() == 0;
if (!done) {
log.info("Topic subscription is not cleaned up yet : {}", stats);
}
return done;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150));
}
}
use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.
the class PulsarFunctionE2ESecurityTest method createFunctionConfig.
protected static FunctionConfig createFunctionConfig(String tenant, String namespace, String functionName, String sourceTopic, String sinkTopic, String subscriptionName) {
FunctionConfig functionConfig = new FunctionConfig();
functionConfig.setTenant(tenant);
functionConfig.setNamespace(namespace);
functionConfig.setName(functionName);
functionConfig.setParallelism(1);
functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE);
functionConfig.setSubName(subscriptionName);
functionConfig.setInputs(Collections.singleton(sourceTopic));
functionConfig.setAutoAck(true);
functionConfig.setClassName("org.apache.pulsar.functions.api.examples.ExclamationFunction");
functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
functionConfig.setOutput(sinkTopic);
functionConfig.setCleanupSubscription(true);
return functionConfig;
}
use of org.apache.pulsar.common.functions.FunctionConfig in project pulsar by apache.
the class PulsarFunctionPublishTest method testUpdateFunctionUserConfig.
@Test
public void testUpdateFunctionUserConfig() throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String publishTopic = "persistent://" + replNamespace + "/publishtopic";
final String functionName = "test-update-user-config";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, publishTopic, subscriptionName);
String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
Map<String, Object> userConfig = functionConfig.getUserConfig();
functionConfig.setUserConfig(null);
admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
FunctionConfig updatefunctionConfig1 = admin.functions().getFunction(tenant, namespacePortion, functionName);
Assert.assertEquals(userConfig, updatefunctionConfig1.getUserConfig());
Map<String, Object> newUserConfig = new HashMap<>();
newUserConfig.put("publish-topic", publishTopic);
newUserConfig.put("test", "test");
updatefunctionConfig1.setUserConfig(newUserConfig);
admin.functions().updateFunctionWithUrl(updatefunctionConfig1, jarFilePathUrl);
FunctionConfig updatefunctionConfig2 = admin.functions().getFunction(tenant, namespacePortion, functionName);
Assert.assertEquals(updatefunctionConfig2.getUserConfig(), updatefunctionConfig1.getUserConfig());
}
Aggregations