use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.
the class PulsarFunctionLocalRunTest method testPulsarSourceLocalRun.
private void testPulsarSourceLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sinkTopic = "persistent://" + replNamespace + "/output";
final String sourceName = "PulsarSource-test";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
SourceConfig sourceConfig = createSourceConfig(tenant, namespacePortion, sourceName, sinkTopic);
if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
sourceConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorSource");
}
sourceConfig.setArchive(jarFilePathUrl);
sourceConfig.setParallelism(parallelism);
int metricsPort = FunctionCommon.findAvailablePort();
@Cleanup LocalRunner localRunner = LocalRunner.builder().sourceConfig(sourceConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
localRunner.start(false);
Assert.assertTrue(retryStrategically((test) -> {
try {
return admin.topics().getStats(sinkTopic).getPublishers().size() == parallelism;
} catch (PulsarAdminException e) {
return false;
}
}, 10, 150));
Assert.assertTrue(retryStrategically((test) -> {
try {
boolean result = false;
TopicStats sourceStats = admin.topics().getStats(sinkTopic);
if (sourceStats.getPublishers().size() == parallelism) {
for (PublisherStats publisher : sourceStats.getPublishers()) {
result = publisher.getMetadata() != null && publisher.getMetadata().containsKey("id") && publisher.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, sourceName));
}
}
return result;
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150));
Assert.assertTrue(retryStrategically((test) -> {
try {
return (admin.topics().getStats(sinkTopic).getPublishers().size() == parallelism) && (admin.topics().getInternalStats(sinkTopic, false).numberOfEntries > 4);
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150));
assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), parallelism);
// validate prometheus metrics
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
log.info("prometheus metrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
if (line.startsWith("pulsar_source_written_total")) {
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
assertFalse(metrics.isEmpty());
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_written_total");
if (m != null) {
metricsMap.put(m.tags.get("instance_id"), m);
}
}
});
Assert.assertEquals(metricsMap.size(), parallelism);
for (int i = 0; i < parallelism; i++) {
PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
Assert.assertNotNull(m);
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), String.valueOf(i));
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertTrue(m.value > 0.0);
}
localRunner.stop();
Assert.assertTrue(retryStrategically((test) -> {
try {
return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
} catch (PulsarAdminException e) {
return e.getStatusCode() == 404;
}
}, 10, 150));
try {
assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
} catch (PulsarAdminException e) {
if (e.getStatusCode() != 404) {
fail();
}
}
}
use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.
the class PulsarFunctionLocalRunTest method createSourceConfig.
private static SourceConfig createSourceConfig(String tenant, String namespace, String functionName, String sinkTopic) {
SourceConfig sourceConfig = new SourceConfig();
sourceConfig.setTenant(tenant);
sourceConfig.setNamespace(namespace);
sourceConfig.setName(functionName);
sourceConfig.setParallelism(1);
sourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
sourceConfig.setTopicName(sinkTopic);
return sourceConfig;
}
use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.
the class PulsarBatchSourceE2ETest method testPulsarBatchSourceStats.
private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception {
final String namespacePortion = "io";
final String replNamespace = tenant + "/" + namespacePortion;
final String sinkTopic = "persistent://" + replNamespace + "/output";
final String sourceName = "PulsarBatchSource";
admin.namespaces().createNamespace(replNamespace);
Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
SourceConfig sourceConfig = createSourceConfig(tenant, namespacePortion, sourceName, sinkTopic);
sourceConfig.setBatchSourceConfig(createBatchSourceConfig());
retryStrategically((test) -> {
try {
return (admin.topics().getStats(sinkTopic).getPublishers().size() == 1);
} catch (PulsarAdminException e) {
return false;
}
}, 10, 150);
final String sinkTopic2 = "persistent://" + replNamespace + "/output-" + sourceName;
sourceConfig.setTopicName(sinkTopic2);
if (jarFilePathUrl.startsWith(Utils.BUILTIN)) {
sourceConfig.setArchive(jarFilePathUrl);
admin.sources().createSource(sourceConfig, jarFilePathUrl);
} else {
admin.sources().createSourceWithUrl(sourceConfig, jarFilePathUrl);
}
retryStrategically((test) -> {
try {
TopicStats sourceStats = admin.topics().getStats(sinkTopic2);
return sourceStats.getPublishers().size() == 1 && sourceStats.getPublishers().get(0).getMetadata() != null && sourceStats.getPublishers().get(0).getMetadata().containsKey("id") && sourceStats.getPublishers().get(0).getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, sourceName));
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
TopicStats sourceStats = admin.topics().getStats(sinkTopic2);
assertEquals(sourceStats.getPublishers().size(), 1);
assertNotNull(sourceStats.getPublishers().get(0).getMetadata());
assertTrue(sourceStats.getPublishers().get(0).getMetadata().containsKey("id"));
assertEquals(sourceStats.getPublishers().get(0).getMetadata().get("id"), String.format("%s/%s/%s", tenant, namespacePortion, sourceName));
retryStrategically((test) -> {
try {
return (admin.topics().getStats(sinkTopic2).getPublishers().size() == 1) && (admin.topics().getInternalStats(sinkTopic2, false).numberOfEntries > 4);
} catch (PulsarAdminException e) {
return false;
}
}, 50, 150);
assertEquals(admin.topics().getStats(sinkTopic2).getPublishers().size(), 1);
String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get());
log.info("prometheusMetrics: {}", prometheusMetrics);
Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics);
PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_received_total");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertTrue(m.value > 0.0);
m = metrics.get("pulsar_source_received_total_1min");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertTrue(m.value > 0.0);
m = metrics.get("pulsar_source_written_total");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertTrue(m.value > 0.0);
m = metrics.get("pulsar_source_written_total_1min");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertTrue(m.value > 0.0);
m = metrics.get("pulsar_source_source_exceptions_total");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertEquals(m.value, 0.0);
m = metrics.get("pulsar_source_source_exceptions_total_1min");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertEquals(m.value, 0.0);
m = metrics.get("pulsar_source_system_exceptions_total");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertEquals(m.value, 0.0);
m = metrics.get("pulsar_source_system_exceptions_total_1min");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertEquals(m.value, 0.0);
m = metrics.get("pulsar_source_last_invocation");
assertEquals(m.tags.get("cluster"), config.getClusterName());
assertEquals(m.tags.get("instance_id"), "0");
assertEquals(m.tags.get("name"), sourceName);
assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
assertTrue(m.value > 0.0);
tempDirectory.assertThatFunctionDownloadTempFilesHaveBeenDeleted();
admin.sources().deleteSource(tenant, namespacePortion, sourceName);
}
use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.
the class PulsarBatchSourceE2ETest method createSourceConfig.
private static SourceConfig createSourceConfig(String tenant, String namespace, String functionName, String sinkTopic) {
SourceConfig sourceConfig = new SourceConfig();
sourceConfig.setTenant(tenant);
sourceConfig.setNamespace(namespace);
sourceConfig.setName(functionName);
sourceConfig.setParallelism(1);
sourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
sourceConfig.setTopicName(sinkTopic);
return sourceConfig;
}
use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.
the class SourcesImpl method createSourceAsync.
@Override
public CompletableFuture<Void> createSourceAsync(SourceConfig sourceConfig, String fileName) {
final CompletableFuture<Void> future = new CompletableFuture<>();
try {
RequestBuilder builder = post(source.path(sourceConfig.getTenant()).path(sourceConfig.getNamespace()).path(sourceConfig.getName()).getUri().toASCIIString()).addBodyPart(new StringPart("sourceConfig", ObjectMapperFactory.getThreadLocal().writeValueAsString(sourceConfig), MediaType.APPLICATION_JSON));
if (fileName != null && !fileName.startsWith("builtin://")) {
// If the function code is built in, we don't need to submit here
builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM));
}
asyncHttpClient.executeRequest(addAuthHeaders(source, builder).build()).toCompletableFuture().thenAccept(response -> {
if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) {
future.completeExceptionally(getApiException(Response.status(response.getStatusCode()).entity(response.getResponseBody()).build()));
} else {
future.complete(null);
}
}).exceptionally(throwable -> {
future.completeExceptionally(getApiException(throwable));
return null;
});
} catch (Exception e) {
future.completeExceptionally(getApiException(e));
}
return future;
}
Aggregations