Search in sources :

Example 1 with SourceConfig

use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.

the class PulsarFunctionLocalRunTest method testPulsarSourceLocalRun.

private void testPulsarSourceLocalRun(String jarFilePathUrl, int parallelism) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String sourceName = "PulsarSource-test";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList(CLUSTER));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    SourceConfig sourceConfig = createSourceConfig(tenant, namespacePortion, sourceName, sinkTopic);
    if (jarFilePathUrl == null || !jarFilePathUrl.endsWith(".nar")) {
        sourceConfig.setClassName("org.apache.pulsar.io.datagenerator.DataGeneratorSource");
    }
    sourceConfig.setArchive(jarFilePathUrl);
    sourceConfig.setParallelism(parallelism);
    int metricsPort = FunctionCommon.findAvailablePort();
    @Cleanup LocalRunner localRunner = LocalRunner.builder().sourceConfig(sourceConfig).clientAuthPlugin(AuthenticationTls.class.getName()).clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)).useTls(true).tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH).tlsAllowInsecureConnection(true).tlsHostNameVerificationEnabled(false).brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).connectorsDirectory(workerConfig.getConnectorsDirectory()).metricsPortStart(metricsPort).build();
    localRunner.start(false);
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sinkTopic).getPublishers().size() == parallelism;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 10, 150));
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            boolean result = false;
            TopicStats sourceStats = admin.topics().getStats(sinkTopic);
            if (sourceStats.getPublishers().size() == parallelism) {
                for (PublisherStats publisher : sourceStats.getPublishers()) {
                    result = publisher.getMetadata() != null && publisher.getMetadata().containsKey("id") && publisher.getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, sourceName));
                }
            }
            return result;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150));
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == parallelism) && (admin.topics().getInternalStats(sinkTopic, false).numberOfEntries > 4);
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150));
    assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), parallelism);
    // validate prometheus metrics
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort);
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metricsMap = new HashMap<>();
    Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> {
        if (line.startsWith("pulsar_source_written_total")) {
            Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(line);
            assertFalse(metrics.isEmpty());
            PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_written_total");
            if (m != null) {
                metricsMap.put(m.tags.get("instance_id"), m);
            }
        }
    });
    Assert.assertEquals(metricsMap.size(), parallelism);
    for (int i = 0; i < parallelism; i++) {
        PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i));
        Assert.assertNotNull(m);
        assertEquals(m.tags.get("cluster"), config.getClusterName());
        assertEquals(m.tags.get("instance_id"), String.valueOf(i));
        assertEquals(m.tags.get("name"), sourceName);
        assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
        assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
        assertTrue(m.value > 0.0);
    }
    localRunner.stop();
    Assert.assertTrue(retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == 0);
        } catch (PulsarAdminException e) {
            return e.getStatusCode() == 404;
        }
    }, 10, 150));
    try {
        assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), 0);
    } catch (PulsarAdminException e) {
        if (e.getStatusCode() != 404) {
            fail();
        }
    }
}
Also used : LocalRunner(org.apache.pulsar.functions.LocalRunner) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) Arrays(java.util.Arrays) URL(java.net.URL) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Cleanup(lombok.Cleanup) Test(org.testng.annotations.Test) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) Sink(org.apache.pulsar.io.core.Sink) AfterMethod(org.testng.annotations.AfterMethod) ByteBuffer(java.nio.ByteBuffer) URLClassLoader(java.net.URLClassLoader) NarClassLoader(org.apache.pulsar.common.nar.NarClassLoader) Map(java.util.Map) MockedPulsarServiceBaseTest.retryStrategically(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically) FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) Assert.assertFalse(org.testng.Assert.assertFalse) Method(java.lang.reflect.Method) Record(org.apache.pulsar.functions.api.Record) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) BeforeClass(org.testng.annotations.BeforeClass) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) Consumer(org.apache.pulsar.client.api.Consumer) FutureUtil(org.apache.pulsar.common.util.FutureUtil) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) LocalBookkeeperEnsemble(org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble) ServiceConfigurationUtils(org.apache.pulsar.broker.ServiceConfigurationUtils) AuthenticationProviderTls(org.apache.pulsar.broker.authentication.AuthenticationProviderTls) PulsarAuthorizationProvider(org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Utils(org.apache.pulsar.common.functions.Utils) DataProvider(org.testng.annotations.DataProvider) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) HashSet(java.util.HashSet) AuthenticationTls(org.apache.pulsar.client.impl.auth.AuthenticationTls) Lists(com.google.common.collect.Lists) NarClassLoaderBuilder(org.apache.pulsar.common.nar.NarClassLoaderBuilder) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Assert(org.testng.Assert) SchemaDefinition(org.apache.pulsar.client.api.schema.SchemaDefinition) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TenantInfo(org.apache.pulsar.common.policies.data.TenantInfo) AfterClass(org.testng.annotations.AfterClass) SourceConfig(org.apache.pulsar.common.io.SourceConfig) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) JAVA_INSTANCE_JAR_PROPERTY(org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Assert.fail(org.testng.Assert.fail) ConsumerConfig(org.apache.pulsar.common.functions.ConsumerConfig) SinkConfig(org.apache.pulsar.common.io.SinkConfig) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) File(java.io.File) Schema(org.apache.pulsar.client.api.Schema) TimeUnit(java.util.concurrent.TimeUnit) GenericRecord(org.apache.pulsar.client.api.schema.GenericRecord) FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon) SinkContext(org.apache.pulsar.io.core.SinkContext) Authentication(org.apache.pulsar.client.api.Authentication) Assert.assertTrue(org.testng.Assert.assertTrue) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) BrokerStats(org.apache.pulsar.client.admin.BrokerStats) Collections(java.util.Collections) LocalRunner(org.apache.pulsar.functions.LocalRunner) HashMap(java.util.HashMap) SourceConfig(org.apache.pulsar.common.io.SourceConfig) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Cleanup(lombok.Cleanup) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) TopicStats(org.apache.pulsar.common.policies.data.TopicStats)

Example 2 with SourceConfig

use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.

the class PulsarFunctionLocalRunTest method createSourceConfig.

private static SourceConfig createSourceConfig(String tenant, String namespace, String functionName, String sinkTopic) {
    SourceConfig sourceConfig = new SourceConfig();
    sourceConfig.setTenant(tenant);
    sourceConfig.setNamespace(namespace);
    sourceConfig.setName(functionName);
    sourceConfig.setParallelism(1);
    sourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sourceConfig.setTopicName(sinkTopic);
    return sourceConfig;
}
Also used : SourceConfig(org.apache.pulsar.common.io.SourceConfig)

Example 3 with SourceConfig

use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.

the class PulsarBatchSourceE2ETest method testPulsarBatchSourceStats.

private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String sourceName = "PulsarBatchSource";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    SourceConfig sourceConfig = createSourceConfig(tenant, namespacePortion, sourceName, sinkTopic);
    sourceConfig.setBatchSourceConfig(createBatchSourceConfig());
    retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic).getPublishers().size() == 1);
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 10, 150);
    final String sinkTopic2 = "persistent://" + replNamespace + "/output-" + sourceName;
    sourceConfig.setTopicName(sinkTopic2);
    if (jarFilePathUrl.startsWith(Utils.BUILTIN)) {
        sourceConfig.setArchive(jarFilePathUrl);
        admin.sources().createSource(sourceConfig, jarFilePathUrl);
    } else {
        admin.sources().createSourceWithUrl(sourceConfig, jarFilePathUrl);
    }
    retryStrategically((test) -> {
        try {
            TopicStats sourceStats = admin.topics().getStats(sinkTopic2);
            return sourceStats.getPublishers().size() == 1 && sourceStats.getPublishers().get(0).getMetadata() != null && sourceStats.getPublishers().get(0).getMetadata().containsKey("id") && sourceStats.getPublishers().get(0).getMetadata().get("id").equals(String.format("%s/%s/%s", tenant, namespacePortion, sourceName));
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    TopicStats sourceStats = admin.topics().getStats(sinkTopic2);
    assertEquals(sourceStats.getPublishers().size(), 1);
    assertNotNull(sourceStats.getPublishers().get(0).getMetadata());
    assertTrue(sourceStats.getPublishers().get(0).getMetadata().containsKey("id"));
    assertEquals(sourceStats.getPublishers().get(0).getMetadata().get("id"), String.format("%s/%s/%s", tenant, namespacePortion, sourceName));
    retryStrategically((test) -> {
        try {
            return (admin.topics().getStats(sinkTopic2).getPublishers().size() == 1) && (admin.topics().getInternalStats(sinkTopic2, false).numberOfEntries > 4);
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    assertEquals(admin.topics().getStats(sinkTopic2).getPublishers().size(), 1);
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get());
    log.info("prometheusMetrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics);
    PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_received_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_source_received_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_source_written_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_source_written_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_source_source_exceptions_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_source_source_exceptions_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_source_system_exceptions_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_source_system_exceptions_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_source_last_invocation");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), sourceName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, sourceName));
    assertTrue(m.value > 0.0);
    tempDirectory.assertThatFunctionDownloadTempFilesHaveBeenDeleted();
    admin.sources().deleteSource(tenant, namespacePortion, sourceName);
}
Also used : PulsarFunctionTestUtils(org.apache.pulsar.functions.worker.PulsarFunctionTestUtils) BatchSourceConfig(org.apache.pulsar.common.io.BatchSourceConfig) SourceConfig(org.apache.pulsar.common.io.SourceConfig) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) TopicStats(org.apache.pulsar.common.policies.data.TopicStats)

Example 4 with SourceConfig

use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.

the class PulsarBatchSourceE2ETest method createSourceConfig.

private static SourceConfig createSourceConfig(String tenant, String namespace, String functionName, String sinkTopic) {
    SourceConfig sourceConfig = new SourceConfig();
    sourceConfig.setTenant(tenant);
    sourceConfig.setNamespace(namespace);
    sourceConfig.setName(functionName);
    sourceConfig.setParallelism(1);
    sourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sourceConfig.setTopicName(sinkTopic);
    return sourceConfig;
}
Also used : BatchSourceConfig(org.apache.pulsar.common.io.BatchSourceConfig) SourceConfig(org.apache.pulsar.common.io.SourceConfig)

Example 5 with SourceConfig

use of org.apache.pulsar.common.io.SourceConfig in project pulsar by apache.

the class SourcesImpl method createSourceAsync.

@Override
public CompletableFuture<Void> createSourceAsync(SourceConfig sourceConfig, String fileName) {
    final CompletableFuture<Void> future = new CompletableFuture<>();
    try {
        RequestBuilder builder = post(source.path(sourceConfig.getTenant()).path(sourceConfig.getNamespace()).path(sourceConfig.getName()).getUri().toASCIIString()).addBodyPart(new StringPart("sourceConfig", ObjectMapperFactory.getThreadLocal().writeValueAsString(sourceConfig), MediaType.APPLICATION_JSON));
        if (fileName != null && !fileName.startsWith("builtin://")) {
            // If the function code is built in, we don't need to submit here
            builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM));
        }
        asyncHttpClient.executeRequest(addAuthHeaders(source, builder).build()).toCompletableFuture().thenAccept(response -> {
            if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) {
                future.completeExceptionally(getApiException(Response.status(response.getStatusCode()).entity(response.getResponseBody()).build()));
            } else {
                future.complete(null);
            }
        }).exceptionally(throwable -> {
            future.completeExceptionally(getApiException(throwable));
            return null;
        });
    } catch (Exception e) {
        future.completeExceptionally(getApiException(e));
    }
    return future;
}
Also used : AsyncHttpClient(org.asynchttpclient.AsyncHttpClient) ConnectorDefinition(org.apache.pulsar.common.io.ConnectorDefinition) UpdateOptions(org.apache.pulsar.common.functions.UpdateOptions) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) SourceStatus(org.apache.pulsar.common.policies.data.SourceStatus) CompletableFuture(java.util.concurrent.CompletableFuture) Source(org.apache.pulsar.client.admin.Source) FormDataMultiPart(org.glassfish.jersey.media.multipart.FormDataMultiPart) MediaType(javax.ws.rs.core.MediaType) Gson(com.google.gson.Gson) InvocationCallback(javax.ws.rs.client.InvocationCallback) FormDataBodyPart(org.glassfish.jersey.media.multipart.FormDataBodyPart) RequestBuilder(org.asynchttpclient.RequestBuilder) SourceConfig(org.apache.pulsar.common.io.SourceConfig) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Sources(org.apache.pulsar.client.admin.Sources) UpdateOptionsImpl(org.apache.pulsar.common.functions.UpdateOptionsImpl) FilePart(org.asynchttpclient.request.body.multipart.FilePart) Entity(javax.ws.rs.client.Entity) File(java.io.File) Dsl.post(org.asynchttpclient.Dsl.post) GenericType(javax.ws.rs.core.GenericType) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Dsl.put(org.asynchttpclient.Dsl.put) Response(javax.ws.rs.core.Response) Authentication(org.apache.pulsar.client.api.Authentication) StringPart(org.asynchttpclient.request.body.multipart.StringPart) WebTarget(javax.ws.rs.client.WebTarget) CompletableFuture(java.util.concurrent.CompletableFuture) RequestBuilder(org.asynchttpclient.RequestBuilder) StringPart(org.asynchttpclient.request.body.multipart.StringPart) File(java.io.File) FilePart(org.asynchttpclient.request.body.multipart.FilePart) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException)

Aggregations

SourceConfig (org.apache.pulsar.common.io.SourceConfig)192 BatchSourceConfig (org.apache.pulsar.common.io.BatchSourceConfig)141 Test (org.testng.annotations.Test)129 Gson (com.google.gson.Gson)42 HashMap (java.util.HashMap)30 File (java.io.File)27 PulsarAdminException (org.apache.pulsar.client.admin.PulsarAdminException)27 Function (org.apache.pulsar.functions.proto.Function)24 IOException (java.io.IOException)21 Response (javax.ws.rs.core.Response)18 List (java.util.List)15 Map (java.util.Map)15 RestException (org.apache.pulsar.common.util.RestException)15 IdentityFunction (org.apache.pulsar.functions.api.utils.IdentityFunction)15 FunctionMetaDataManager (org.apache.pulsar.functions.worker.FunctionMetaDataManager)15 Lists (com.google.common.collect.Lists)12 FileInputStream (java.io.FileInputStream)12 InputStream (java.io.InputStream)12 Files (java.nio.file.Files)12 PulsarAdmin (org.apache.pulsar.client.admin.PulsarAdmin)12