use of org.apache.pulsar.io.core.Sink in project incubator-pulsar by apache.
the class JavaInstanceRunnable method setupOutput.
private void setupOutput(ContextImpl contextImpl) throws Exception {
SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
Object object;
// If sink classname is not set, we default pulsar sink
if (sinkSpec.getClassName().isEmpty()) {
if (StringUtils.isEmpty(sinkSpec.getTopic())) {
object = PulsarSinkDisable.INSTANCE;
} else {
PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
pulsarSinkConfig.setTopic(sinkSpec.getTopic());
pulsarSinkConfig.setForwardSourceMessageProperty(this.instanceConfig.getFunctionDetails().getSink().getForwardSourceMessageProperty());
if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
pulsarSinkConfig.setSchemaType(sinkSpec.getSchemaType());
} else if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
pulsarSinkConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
}
pulsarSinkConfig.setTypeClassName(sinkSpec.getTypeClassName());
pulsarSinkConfig.setSchemaProperties(sinkSpec.getSchemaPropertiesMap());
if (this.instanceConfig.getFunctionDetails().getSink().getProducerSpec() != null) {
org.apache.pulsar.functions.proto.Function.ProducerSpec conf = this.instanceConfig.getFunctionDetails().getSink().getProducerSpec();
ProducerConfig.ProducerConfigBuilder builder = ProducerConfig.builder().maxPendingMessages(conf.getMaxPendingMessages()).maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()).batchBuilder(conf.getBatchBuilder()).useThreadLocalProducers(conf.getUseThreadLocalProducers()).cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
pulsarSinkConfig.setProducerConfig(builder.build());
}
object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, this.functionClassLoader);
}
} else {
object = Reflections.createInstance(sinkSpec.getClassName(), this.functionClassLoader);
}
if (object instanceof Sink) {
this.sink = (Sink) object;
} else {
throw new RuntimeException("Sink does not implement correct interface");
}
if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
Thread.currentThread().setContextClassLoader(this.functionClassLoader);
}
try {
if (sinkSpec.getConfigs().isEmpty()) {
if (log.isDebugEnabled()) {
log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString());
}
this.sink.open(new HashMap<>(), contextImpl);
} else {
if (log.isDebugEnabled()) {
log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec, contextImpl.toString());
}
this.sink.open(ObjectMapperFactory.getThreadLocal().readValue(sinkSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
}), contextImpl);
}
} catch (Exception e) {
log.error("Sink open produced uncaught exception: ", e);
throw e;
} finally {
Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
}
}
use of org.apache.pulsar.io.core.Sink in project pulsar by yahoo.
the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.
private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// create a producer that creates a topic at broker
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
if (className != null) {
sinkConfig.setClassName(className);
} else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
}
sinkConfig.setArchive(jarFilePathUrl);
sinkConfig.setParallelism(parallelism);
int metricsPort = FunctionCommon.findAvailablePort();
@Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
localRunner.start(false);
Assert.assertTrue(retryStrategically((test) -> {
try {
boolean result = false;
TopicStats topicStats = admin.topics().getStats(sourceTopic);
if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
result = consumerStats.getAvailablePermits() == 1000;
}
}
return result;
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
int totalMsgs = 10;
for (int i = 0; i < totalMsgs; i++) {
String data = "my-message-" + i;
producer.newMessage().property(propertyKey, propertyValue).value(data).send();
}
Assert.assertTrue(retryStrategically((test) -> {
try {
SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
} catch (PulsarAdminException e) {
return false;
}
}, 5, 200));
// validate prometheus metrics
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
log.info("prometheus metrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
if (line.startsWith("pulsar_sink_written_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
if (m != null) {
metricsMap.put(m.tags.get("instance_id"), m);
}
} else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
if (m == null) {
m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
}
assertEquals(m.value, 0);
}
});
Assert.assertEquals(metricsMap.size(), parallelism);
double totalNumRecvMsg = 0;
for (int i = 0; i < parallelism; i++) {
PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
Assert.assertNotNull(m);
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), String.valueOf(i));
assertEquals(m.tags.get("name"), sinkName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
totalNumRecvMsg += m.value;
}
assertEquals(totalNumRecvMsg, totalMsgs);
// stop sink
localRunner.stop();
Assert.assertTrue(retryStrategically((test) -> {
try {
TopicStats stats = admin.topics().getStats(sourceTopic);
return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
TopicStats topicStats = admin.topics().getStats(sourceTopic);
assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}
use of org.apache.pulsar.io.core.Sink in project pulsar by apache.
the class JavaInstanceRunnable method setupOutput.
private void setupOutput(ContextImpl contextImpl) throws Exception {
SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
Object object;
// If sink classname is not set, we default pulsar sink
if (sinkSpec.getClassName().isEmpty()) {
if (StringUtils.isEmpty(sinkSpec.getTopic())) {
object = PulsarSinkDisable.INSTANCE;
} else {
PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
pulsarSinkConfig.setTopic(sinkSpec.getTopic());
pulsarSinkConfig.setForwardSourceMessageProperty(this.instanceConfig.getFunctionDetails().getSink().getForwardSourceMessageProperty());
if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
pulsarSinkConfig.setSchemaType(sinkSpec.getSchemaType());
} else if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
pulsarSinkConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
}
pulsarSinkConfig.setTypeClassName(sinkSpec.getTypeClassName());
pulsarSinkConfig.setSchemaProperties(sinkSpec.getSchemaPropertiesMap());
if (this.instanceConfig.getFunctionDetails().getSink().getProducerSpec() != null) {
org.apache.pulsar.functions.proto.Function.ProducerSpec conf = this.instanceConfig.getFunctionDetails().getSink().getProducerSpec();
ProducerConfig.ProducerConfigBuilder builder = ProducerConfig.builder().maxPendingMessages(conf.getMaxPendingMessages()).maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()).batchBuilder(conf.getBatchBuilder()).useThreadLocalProducers(conf.getUseThreadLocalProducers()).cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
pulsarSinkConfig.setProducerConfig(builder.build());
}
object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, this.functionClassLoader);
}
} else {
object = Reflections.createInstance(sinkSpec.getClassName(), this.functionClassLoader);
}
if (object instanceof Sink) {
this.sink = (Sink) object;
} else {
throw new RuntimeException("Sink does not implement correct interface");
}
if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
Thread.currentThread().setContextClassLoader(this.functionClassLoader);
}
try {
if (sinkSpec.getConfigs().isEmpty()) {
if (log.isDebugEnabled()) {
log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString());
}
this.sink.open(new HashMap<>(), contextImpl);
} else {
if (log.isDebugEnabled()) {
log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec.toString(), contextImpl.toString());
}
this.sink.open(ObjectMapperFactory.getThreadLocal().readValue(sinkSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
}), contextImpl);
}
} catch (Exception e) {
log.error("Sink open produced uncaught exception: ", e);
throw e;
} finally {
Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
}
}
use of org.apache.pulsar.io.core.Sink in project pulsar by apache.
the class PulsarFunctionLocalRunTest method testPulsarSinkLocalRun.
private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, String className) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sourceTopic = "persistent://" + replNamespace + "/input";
final String sinkName = "PulsarSink-test";
final String propertyKey = "key";
final String propertyValue = "value";
final String subscriptionName = "test-sub";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("local"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
// create a producer that creates a topic at broker
Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
if (className != null) {
sinkConfig.setClassName(className);
} else if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
sinkConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorPrintSink");
}
sinkConfig.setArchive(jarFilePathUrl);
sinkConfig.setParallelism(parallelism);
int metricsPort = FunctionCommon.findAvailablePort();
@Cleanup LocalRunner localRunner = LocalRunner.builder().sinkConfig(sinkConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
localRunner.start(false);
Assert.assertTrue(retryStrategically((test) -> {
try {
boolean result = false;
TopicStats topicStats = admin.topics().getStats(sourceTopic);
if (topicStats.getSubscriptions().containsKey(subscriptionName) && topicStats.getSubscriptions().get(subscriptionName).getConsumers().size() == parallelism) {
for (ConsumerStats consumerStats : topicStats.getSubscriptions().get(subscriptionName).getConsumers()) {
result = consumerStats.getAvailablePermits() == 1000;
}
}
return result;
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
int totalMsgs = 10;
for (int i = 0; i < totalMsgs; i++) {
String data = "my-message-" + i;
producer.newMessage().property(propertyKey, propertyValue).value(data).send();
}
Assert.assertTrue(retryStrategically((test) -> {
try {
SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
} catch (PulsarAdminException e) {
return false;
}
}, 5, 200));
// validate prometheus metrics
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
log.info("prometheus metrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
if (line.startsWith("pulsar_sink_written_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total");
if (m != null) {
metricsMap.put(m.tags.get("instance_id"), m);
}
} else if (line.startsWith("pulsar_sink_sink_exceptions_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total");
if (m == null) {
m = metrics.get("pulsar_sink_sink_exceptions_total_1min");
}
assertEquals(m.value, 0);
}
});
Assert.assertEquals(metricsMap.size(), parallelism);
double totalNumRecvMsg = 0;
for (int i = 0; i < parallelism; i++) {
PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
Assert.assertNotNull(m);
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), String.valueOf(i));
assertEquals(m.tags.get("name"), sinkName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sinkName));
totalNumRecvMsg += m.value;
}
assertEquals(totalNumRecvMsg, totalMsgs);
// stop sink
localRunner.stop();
Assert.assertTrue(retryStrategically((test) -> {
try {
TopicStats stats = admin.topics().getStats(sourceTopic);
return stats.getSubscriptions().get(subscriptionName) != null && stats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty();
} catch (PulsarAdminException e) {
return false;
}
}, 20, 150));
TopicStats topicStats = admin.topics().getStats(sourceTopic);
assertTrue(topicStats.getSubscriptions().get(subscriptionName) != null && topicStats.getSubscriptions().get(subscriptionName).getConsumers().isEmpty());
}
Aggregations