use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.
the class PulsarFunctionLocalRunTest method testAvroFunctionLocalRun.
private void testAvroFunctionLocalRun(String jarFilePathUrl) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
final String sinkTopic = "persistent://" + replNamespace + "/output";
final String propertyKey = "key";
final String propertyValue = "value";
final String functionName = "PulsarFunction-test";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
Schema schema = Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull(true).withJSR310ConversionEnabled(true).withPojo(avroTestObjectClass).build());
// use AVRO schema
admin.schemas().createSchema(sourceTopic, schema.getSchemaInfo());
// please note that in this test the sink topic schema is different from the schema of the source topic
// produce message to sourceTopic
Producer<Object> producer = pulsarClient.newProducer(schema).topic(sourceTopic).create();
// consume message from sinkTopic
Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic(sinkTopic).subscriptionName("sub").subscribe();
FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopic, sinkTopic, subscriptionName);
// set jsr310ConversionEnabled、alwaysAllowNull
Map<String, String> schemaInput = new HashMap<>();
schemaInput.put(sourceTopic, "{\"schemaType\":\"AVRO\",\"schemaProperties\":{\"__jsr310ConversionEnabled\":\"true\",\"__alwaysAllowNull\":\"true\"}}");
Map<String, String> schemaOutput = new HashMap<>();
schemaOutput.put(sinkTopic, "{\"schemaType\":\"AVRO\",\"schemaProperties\":{\"__jsr310ConversionEnabled\":\"true\",\"__alwaysAllowNull\":\"true\"}}");
functionConfig.setCustomSchemaInputs(schemaInput);
functionConfig.setCustomSchemaOutputs(schemaOutput);
functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
if (jarFilePathUrl == null) {
functionConfig.setClassName("org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction");
} else {
functionConfig.setJar(jarFilePathUrl);
}
@Cleanup LocalRunner localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
localRunner.start(false);
retryStrategically((test) -> {
try {
TopicStats stats = admin.topics().getStats(sourceTopic);
return stats.getSubscriptions().get(subscriptionName) != null && !stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
int totalMsgs = 5;
Method setBaseValueMethod = avroTestObjectClass.getMethod("setBaseValue", new Class[] { int.class });
for (int i = 0; i < totalMsgs; i++) {
Object avroTestObject = avroTestObjectClass.getDeclaredConstructor().newInstance();
setBaseValueMethod.invoke(avroTestObject, i);
producer.newMessage().property(propertyKey, propertyValue).value(avroTestObject).send();
}
// consume message from sinkTopic
for (int i = 0; i < totalMsgs; i++) {
Message<GenericRecord> msg = consumer.receive(5, TimeUnit.SECONDS);
String receivedPropertyValue = msg.getProperty(propertyKey);
assertEquals(propertyValue, receivedPropertyValue);
assertEquals(msg.getValue().getField("baseValue"), 10 + i);
consumer.acknowledge(msg);
}
// validate pulsar-sink consumer has consumed all messages
assertNotEquals(admin.topics().getStats(sinkTopic).getSubscriptions().values().iterator().next().getUnackedMessages(), 0);
localRunner.stop();
retryStrategically((test) -> {
try {
TopicStats topicStats = admin.topics().getStats(sourceTopic);
return topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150);
// change the schema, the function should not run, resulting in no messages to consume
schemaInput.put(sourceTopic, "{\"schemaType\":\"AVRO\",\"schemaProperties\":{\"__jsr310ConversionEnabled\":\"false\",\"__alwaysAllowNull\":\"false\"}}");
localRunner = LocalRunner.builder().functionConfig(functionConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
localRunner.start(false);
producer.newMessage().property(propertyKey, propertyValue).value(avroTestObjectClass.getDeclaredConstructor().newInstance()).send();
Message<GenericRecord> msg = consumer.receive(2, TimeUnit.SECONDS);
Assert.assertNull(msg);
producer.close();
consumer.close();
localRunner.stop();
}
use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.
the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.
private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// create a producer that creates a topic at broker
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
if (className != null) {
sinkConfig.setClassName(className);
} else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
}
sinkConfig.setArchive(jarFilePathUrl);
sinkConfig.setParallelism(parallelism);
int metricsPort = FunctionCommon.findAvailablePort();
@Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
localRunner.start(false);
Assert.assertTrue(retryStrategically((test) -> {
try {
boolean result = false;
TopicStats topicStats = admin.topics().getStats(sourceTopic);
if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
result = consumerStats.getAvailablePermits() == 1000;
}
}
return result;
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
int totalMsgs = 10;
for (int i = 0; i < totalMsgs; i++) {
String data = "my-message-" + i;
producer.newMessage().property(propertyKey, propertyValue).value(data).send();
}
Assert.assertTrue(retryStrategically((test) -> {
try {
SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
} catch (PulsarAdminException e) {
return false;
}
}, 5, 200));
// validate prometheus metrics
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
log.info("prometheus metrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
if (line.startsWith("pulsar_sink_written_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
if (m != null) {
metricsMap.put(m.tags.get("instance_id"), m);
}
} else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
if (m == null) {
m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
}
assertEquals(m.value, 0);
}
});
Assert.assertEquals(metricsMap.size(), parallelism);
double totalNumRecvMsg = 0;
for (int i = 0; i < parallelism; i++) {
PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
Assert.assertNotNull(m);
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), String.valueOf(i));
assertEquals(m.tags.get("name"), sinkName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
totalNumRecvMsg += m.value;
}
assertEquals(totalNumRecvMsg, totalMsgs);
// stop sink
localRunner.stop();
Assert.assertTrue(retryStrategically((test) -> {
try {
TopicStats stats = admin.topics().getStats(sourceTopic);
return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
TopicStats topicStats = admin.topics().getStats(sourceTopic);
assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}
use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.
the class PulsarSinkE2ETest method testPulsarSinkDLQ.
@Test(timeOut = 30000)
public void testPulsarSinkDLQ() throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String dlqTopic = sourceTopic + "-DLQ";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// 1 create producer、DLQ consumer
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();
// 2 setup sink
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
sinkConfig.setMaxMessageRetries(2);
sinkConfig.setDeadLetterTopic(dlqTopic);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
sinkConfig.setClassName(SinkForTest.class.getName());
@Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();
localRunner.start(false);
retryStrategically((test) -> {
try {
TopicStats topicStats = admin.topics().getStats(sourceTopic);
return topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == 1 && topicStats.getSubscriptions().get(subscriptionName).getConsumers().get(0).getAvailablePermits() == 1000;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
// 3 send message
int totalMsgs = 10;
Set<String> remainingMessagesToReceive = new HashSet<>();
for (int i = 0; i < totalMsgs; i++) {
String messageBody = "fail" + i;
producer.newMessage().property(propertyKey, propertyValue).value(messageBody).send();
remainingMessagesToReceive.add(messageBody);
}
// 4 All messages should enter DLQ
for (int i = 0; i < totalMsgs; i++) {
Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
assertNotNull(message);
remainingMessagesToReceive.remove(message.getValue());
}
assertEquals(remainingMessagesToReceive, Collections.emptySet());
// clean up
producer.close();
consumer.close();
}
use of org.apache.pulsar.functions.LocalRunner in project incubator-pulsar by apache.
the class BatchDataGeneratorExec method main.
public static void main(final String[] args) throws Exception {
final String cronString = "0 0/5 * * * ?";
final Map<String, Object> discoveryConfig = new HashMap<>();
discoveryConfig.put(CronTriggerer.CRON_KEY, cronString);
final BatchSourceConfig batchSourceConfig = BatchSourceConfig.builder().discoveryTriggererClassName(CronTriggerer.class.getName()).discoveryTriggererConfig(discoveryConfig).build();
final SourceConfig sourceConfig = SourceConfig.builder().batchSourceConfig(batchSourceConfig).className(BatchDataGeneratorSource.class.getName()).configs(new HashMap<>()).name("BatchDataGenerator").parallelism(1).topicName("persistent://public/default/batchdatagenerator").build();
final LocalRunner localRunner = LocalRunner.builder().brokerServiceUrl("pulsar://localhost:6650").sourceConfig(sourceConfig).build();
localRunner.start(false);
TimeUnit.MINUTES.sleep(30);
localRunner.stop();
System.exit(0);
}
use of org.apache.pulsar.functions.LocalRunner in project pulsar by yahoo.
the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.
private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// create a producer that creates a topic at broker
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
if (className != null) {
sinkConfig.setClassName(className);
} else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
}
sinkConfig.setArchive(jarFilePathUrl);
sinkConfig.setParallelism(parallelism);
int metricsPort = FunctionCommon.findAvailablePort();
@Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
localRunner.start(false);
Assert.assertTrue(retryStrategically((test) -> {
try {
boolean result = false;
TopicStats topicStats = admin.topics().getStats(sourceTopic);
if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
result = consumerStats.getAvailablePermits() == 1000;
}
}
return result;
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
int totalMsgs = 10;
for (int i = 0; i < totalMsgs; i++) {
String data = "my-message-" + i;
producer.newMessage().property(propertyKey, propertyValue).value(data).send();
}
Assert.assertTrue(retryStrategically((test) -> {
try {
SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
} catch (PulsarAdminException e) {
return false;
}
}, 5, 200));
// validate prometheus metrics
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
log.info("prometheus metrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
if (line.startsWith("pulsar_sink_written_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
if (m != null) {
metricsMap.put(m.tags.get("instance_id"), m);
}
} else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
if (m == null) {
m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
}
assertEquals(m.value, 0);
}
});
Assert.assertEquals(metricsMap.size(), parallelism);
double totalNumRecvMsg = 0;
for (int i = 0; i < parallelism; i++) {
PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
Assert.assertNotNull(m);
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), String.valueOf(i));
assertEquals(m.tags.get("name"), sinkName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
totalNumRecvMsg += m.value;
}
assertEquals(totalNumRecvMsg, totalMsgs);
// stop sink
localRunner.stop();
Assert.assertTrue(retryStrategically((test) -> {
try {
TopicStats stats = admin.topics().getStats(sourceTopic);
return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
TopicStats topicStats = admin.topics().getStats(sourceTopic);
assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}
Aggregations