Example 1 with Functions
use of org.apache.pulsar.functions.utils.functions.Functions in project pulsar by apache.
the class LocalRunner method start.
public void start(boolean blocking) throws Exception {
List<RuntimeSpawner> local = new LinkedList<>();
synchronized (this) {
if (!running.compareAndSet(false, true)) {
throw new IllegalArgumentException("Pulsar Function local run already started!");
}
Runtime.getRuntime().addShutdownHook(shutdownHook);
Function.FunctionDetails functionDetails;
String userCodeFile = null;
int parallelism;
if (functionConfig != null) {
FunctionConfigUtils.inferMissingArguments(functionConfig, true);
parallelism = functionConfig.getParallelism();
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
userCodeFile = functionConfig.getJar();
boolean isBuiltin = !StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(Utils.BUILTIN);
if (isBuiltin) {
WorkerConfig workerConfig = WorkerConfig.load(System.getenv("PULSAR_HOME") + "/conf/functions_worker.yml");
Functions functions = FunctionUtils.searchForFunctions(System.getenv("PULSAR_HOME") + workerConfig.getFunctionsDirectory().replaceFirst("^.", ""));
String functionType = functionConfig.getJar().replaceFirst("^builtin://", "");
userCodeFile = functions.getFunctions().get(functionType).toString();
}
if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("User jar does not exist");
}
userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The jar property must be specified in FunctionConfig.");
}
FunctionConfigUtils.validateJavaFunction(functionConfig, Thread.currentThread().getContextClassLoader());
}
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
userCodeFile = functionConfig.getGo();
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) {
userCodeFile = functionConfig.getPy();
} else {
throw new UnsupportedOperationException();
}
functionDetails = FunctionConfigUtils.convert(functionConfig, userCodeClassLoader != null ? userCodeClassLoader : Thread.currentThread().getContextClassLoader());
} else if (sourceConfig != null) {
inferMissingArguments(sourceConfig);
userCodeFile = sourceConfig.getArchive();
parallelism = sourceConfig.getParallelism();
ClassLoader builtInSourceClassLoader = userCodeFile != null ? isBuiltInSource(userCodeFile) : null;
if (builtInSourceClassLoader != null) {
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, builtInSourceClassLoader, true));
userCodeClassLoader = builtInSourceClassLoader;
} else if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
userCodeClassLoader = sourceClassLoader;
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("Source archive (" + userCodeFile + ") does not exist");
}
ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
userCodeClassLoader = sourceClassLoader;
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The archive property must be specified in SourceConfig.");
}
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, Thread.currentThread().getContextClassLoader(), true));
}
} else if (sinkConfig != null) {
inferMissingArguments(sinkConfig);
userCodeFile = sinkConfig.getArchive();
parallelism = sinkConfig.getParallelism();
ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null;
if (builtInSinkClassLoader != null) {
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, builtInSinkClassLoader, true));
userCodeClassLoader = builtInSinkClassLoader;
} else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
userCodeClassLoader = sinkClassLoader;
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("Sink archive does not exist");
}
ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
userCodeClassLoader = sinkClassLoader;
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The archive property must be specified in SourceConfig.");
}
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, Thread.currentThread().getContextClassLoader(), true));
}
} else {
throw new IllegalArgumentException("Must specify Function, Source or Sink config");
}
if (System.getProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) == null) {
System.setProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY, LocalRunner.class.getProtectionDomain().getCodeSource().getLocation().getFile());
}
AuthenticationConfig authConfig = AuthenticationConfig.builder().clientAuthenticationPlugin(clientAuthPlugin).clientAuthenticationParameters(clientAuthParams).useTls(useTls).tlsAllowInsecureConnection(tlsAllowInsecureConnection).tlsHostnameVerificationEnable(tlsHostNameVerificationEnabled).tlsTrustCertsFilePath(tlsTrustCertFilePath).build();
String serviceUrl = DEFAULT_SERVICE_URL;
if (brokerServiceUrl != null) {
serviceUrl = brokerServiceUrl;
}
if (webServiceUrl == null) {
webServiceUrl = DEFAULT_WEB_SERVICE_URL;
}
if ((sourceConfig != null || sinkConfig != null || functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) && (runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
// By default run java functions as threads
startThreadedMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
} else {
startProcessMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
}
local.addAll(spawners);
}
if (blocking) {
if (exitOnError) {
for (RuntimeSpawner spawner : local) {
spawner.join();
log.info("RuntimeSpawner quit because of", spawner.getRuntime().getDeathException());
}
close();
} else {
synchronized (this) {
while (running.get()) {
this.wait();
}
}
}
}
}
Also used :
Functions(org.apache.pulsar.functions.utils.functions.Functions)
LinkedList(java.util.LinkedList)
Function(org.apache.pulsar.functions.proto.Function)
AuthenticationConfig(org.apache.pulsar.functions.instance.AuthenticationConfig)
WorkerConfig(org.apache.pulsar.functions.worker.WorkerConfig)
File(java.io.File)
RuntimeSpawner(org.apache.pulsar.functions.runtime.RuntimeSpawner)
Example 2 with Functions
use of org.apache.pulsar.functions.utils.functions.Functions in project pulsar by yahoo.
the class ComponentImpl method getStreamingOutput.
private StreamingOutput getStreamingOutput(String pkgPath) {
final StreamingOutput streamingOutput = output -> {
if (pkgPath.startsWith(Utils.HTTP)) {
URL url = URI.create(pkgPath).toURL();
try (InputStream inputStream = url.openStream()) {
IOUtils.copy(inputStream, output);
}
} else if (pkgPath.startsWith(Utils.FILE)) {
URI url = URI.create(pkgPath);
File file = new File(url.getPath());
Files.copy(file.toPath(), output);
} else if (pkgPath.startsWith(Utils.BUILTIN) && !worker().getWorkerConfig().getUploadBuiltinSinksSources()) {
String sType = pkgPath.replaceFirst("^builtin://", "");
final String connectorsDir = worker().getWorkerConfig().getConnectorsDirectory();
log.warn("Processing package {} ; looking at the dir {}", pkgPath, connectorsDir);
Functions sinksOrSources = FunctionUtils.searchForFunctions(connectorsDir, true);
Path narPath = sinksOrSources.getFunctions().get(sType);
if (narPath == null) {
throw new IllegalStateException("Didn't find " + pkgPath + " in " + connectorsDir);
}
log.info("Loading {} from {}", pkgPath, narPath);
try (InputStream in = new FileInputStream(narPath.toString())) {
IOUtils.copy(in, output, 1024);
output.flush();
}
} else if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) {
try {
File file = downloadPackageFile(worker(), pkgPath);
try (InputStream in = new FileInputStream(file)) {
IOUtils.copy(in, output, 1024);
output.flush();
}
} catch (Exception e) {
log.error("Failed download package {} from packageMangment Service", pkgPath, e);
}
} else {
WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), output, pkgPath);
}
};
return streamingOutput;
}
Also used :
RestUtils.throwUnavailableException(org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException)
Functions(org.apache.pulsar.functions.utils.functions.Functions)
FormDataContentDisposition(org.glassfish.jersey.media.multipart.FormDataContentDisposition)
StorageClient(org.apache.bookkeeper.api.StorageClient)
URL(java.net.URL)
Producer(org.apache.pulsar.client.api.Producer)
SchemaSerializationException(org.apache.pulsar.client.api.SchemaSerializationException)
FunctionMetaData(org.apache.pulsar.functions.proto.Function.FunctionMetaData)
Unpooled(io.netty.buffer.Unpooled)
StorageAdminClient(org.apache.bookkeeper.clients.admin.StorageAdminClient)
FunctionCommon.getStateNamespace(org.apache.pulsar.functions.utils.FunctionCommon.getStateNamespace)
StorageClientSettings(org.apache.bookkeeper.clients.config.StorageClientSettings)
FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails)
FunctionConfigUtils(org.apache.pulsar.functions.utils.FunctionConfigUtils)
FunctionCommon.isFunctionCodeBuiltin(org.apache.pulsar.functions.utils.FunctionCommon.isFunctionCodeBuiltin)
PulsarWorkerService(org.apache.pulsar.functions.worker.PulsarWorkerService)
FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig)
NamespaceName(org.apache.pulsar.common.naming.NamespaceName)
UriBuilder(javax.ws.rs.core.UriBuilder)
KeyValue(org.apache.bookkeeper.api.kv.result.KeyValue)
URI(java.net.URI)
WorkerInfo(org.apache.pulsar.common.functions.WorkerInfo)
StringUtils.isEmpty(org.apache.commons.lang3.StringUtils.isEmpty)
Path(java.nio.file.Path)
FunctionMetaDataUtils(org.apache.pulsar.functions.utils.FunctionMetaDataUtils)
Function(org.apache.pulsar.functions.proto.Function)
FunctionCommon.createPkgTempFile(org.apache.pulsar.functions.utils.FunctionCommon.createPkgTempFile)
WorkerUtils(org.apache.pulsar.functions.worker.WorkerUtils)
Collection(java.util.Collection)
StreamingOutput(javax.ws.rs.core.StreamingOutput)
StorageClientBuilder(org.apache.bookkeeper.clients.StorageClientBuilder)
PackageMetadata(org.apache.pulsar.packages.management.core.common.PackageMetadata)
AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource)
IOUtils(org.apache.commons.io.IOUtils)
FunctionMetaDataManager(org.apache.pulsar.functions.worker.FunctionMetaDataManager)
Base64(java.util.Base64)
List(java.util.List)
Slf4j(lombok.extern.slf4j.Slf4j)
StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank)
Response(javax.ws.rs.core.Response)
WebApplicationException(javax.ws.rs.WebApplicationException)
Table(org.apache.bookkeeper.api.kv.Table)
TenantInfoImpl(org.apache.pulsar.common.policies.data.TenantInfoImpl)
PackageName(org.apache.pulsar.packages.management.core.common.PackageName)
Utils(org.apache.pulsar.common.functions.Utils)
SourceSpec(org.apache.pulsar.functions.proto.Function.SourceSpec)
TopicName(org.apache.pulsar.common.naming.TopicName)
RuntimeSpawner(org.apache.pulsar.functions.runtime.RuntimeSpawner)
ConnectorDefinition(org.apache.pulsar.common.io.ConnectorDefinition)
NamespaceNotFoundException(org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException)
InstanceCommunication(org.apache.pulsar.functions.proto.InstanceCommunication)
Message(org.apache.pulsar.client.api.Message)
AtomicReference(java.util.concurrent.atomic.AtomicReference)
Supplier(java.util.function.Supplier)
RestException(org.apache.pulsar.common.util.RestException)
FunctionsImpl.downloadPackageFile(org.apache.pulsar.functions.worker.rest.api.FunctionsImpl.downloadPackageFile)
ByteBuf(io.netty.buffer.ByteBuf)
FunctionInstanceStatsDataImpl(org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl)
FunctionCommon.getUniquePackageName(org.apache.pulsar.functions.utils.FunctionCommon.getUniquePackageName)
LinkedList(java.util.LinkedList)
FunctionsImpl(org.apache.pulsar.client.admin.internal.FunctionsImpl)
Status(javax.ws.rs.core.Response.Status)
FunctionState(org.apache.pulsar.common.functions.FunctionState)
ComponentTypeUtils(org.apache.pulsar.functions.utils.ComponentTypeUtils)
InstanceUtils(org.apache.pulsar.functions.instance.InstanceUtils)
StreamNotFoundException(org.apache.bookkeeper.clients.exceptions.StreamNotFoundException)
Files(java.nio.file.Files)
UTF_8(java.nio.charset.StandardCharsets.UTF_8)
PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException)
FunctionUtils(org.apache.pulsar.functions.utils.functions.FunctionUtils)
Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull)
PackageLocationMetaData(org.apache.pulsar.functions.proto.Function.PackageLocationMetaData)
FileOutputStream(java.io.FileOutputStream)
Reader(org.apache.pulsar.client.api.Reader)
FunctionRuntimeInfo(org.apache.pulsar.functions.worker.FunctionRuntimeInfo)
IOException(java.io.IOException)
FileInputStream(java.io.FileInputStream)
File(java.io.File)
Schema(org.apache.pulsar.client.api.Schema)
ExecutionException(java.util.concurrent.ExecutionException)
TimeUnit(java.util.concurrent.TimeUnit)
ByteBufUtil(io.netty.buffer.ByteBufUtil)
MessageId(org.apache.pulsar.client.api.MessageId)
FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon)
FutureUtils.result(org.apache.bookkeeper.common.concurrent.FutureUtils.result)
FunctionStatsImpl(org.apache.pulsar.common.policies.data.FunctionStatsImpl)
Codec(org.apache.pulsar.common.util.Codec)
WorkerService(org.apache.pulsar.functions.worker.WorkerService)
SinkSpec(org.apache.pulsar.functions.proto.Function.SinkSpec)
FunctionRuntimeManager(org.apache.pulsar.functions.worker.FunctionRuntimeManager)
SECONDS(java.util.concurrent.TimeUnit.SECONDS)
InputStream(java.io.InputStream)
Component(org.apache.pulsar.functions.worker.service.api.Component)
Path(java.nio.file.Path)
FileInputStream(java.io.FileInputStream)
InputStream(java.io.InputStream)
StreamingOutput(javax.ws.rs.core.StreamingOutput)
Functions(org.apache.pulsar.functions.utils.functions.Functions)
URI(java.net.URI)
FunctionCommon.createPkgTempFile(org.apache.pulsar.functions.utils.FunctionCommon.createPkgTempFile)
FunctionsImpl.downloadPackageFile(org.apache.pulsar.functions.worker.rest.api.FunctionsImpl.downloadPackageFile)
File(java.io.File)
URL(java.net.URL)
FileInputStream(java.io.FileInputStream)
RestUtils.throwUnavailableException(org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException)
SchemaSerializationException(org.apache.pulsar.client.api.SchemaSerializationException)
WebApplicationException(javax.ws.rs.WebApplicationException)
NamespaceNotFoundException(org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException)
RestException(org.apache.pulsar.common.util.RestException)
StreamNotFoundException(org.apache.bookkeeper.clients.exceptions.StreamNotFoundException)
PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException)
IOException(java.io.IOException)
ExecutionException(java.util.concurrent.ExecutionException)
Example 3 with Functions
use of org.apache.pulsar.functions.utils.functions.Functions in project pulsar by yahoo.
the class LocalRunner method start.
public void start(boolean blocking) throws Exception {
List<RuntimeSpawner> local = new LinkedList<>();
synchronized (this) {
if (!running.compareAndSet(false, true)) {
throw new IllegalArgumentException("Pulsar Function local run already started!");
}
Runtime.getRuntime().addShutdownHook(shutdownHook);
Function.FunctionDetails functionDetails;
String userCodeFile = null;
int parallelism;
if (functionConfig != null) {
FunctionConfigUtils.inferMissingArguments(functionConfig, true);
parallelism = functionConfig.getParallelism();
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
userCodeFile = functionConfig.getJar();
boolean isBuiltin = !StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(Utils.BUILTIN);
if (isBuiltin) {
WorkerConfig workerConfig = WorkerConfig.load(System.getenv("PULSAR_HOME") + "/conf/functions_worker.yml");
Functions functions = FunctionUtils.searchForFunctions(System.getenv("PULSAR_HOME") + workerConfig.getFunctionsDirectory().replaceFirst("^.", ""));
String functionType = functionConfig.getJar().replaceFirst("^builtin://", "");
userCodeFile = functions.getFunctions().get(functionType).toString();
}
if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("User jar does not exist");
}
userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The jar property must be specified in FunctionConfig.");
}
FunctionConfigUtils.validateJavaFunction(functionConfig, Thread.currentThread().getContextClassLoader());
}
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
userCodeFile = functionConfig.getGo();
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) {
userCodeFile = functionConfig.getPy();
} else {
throw new UnsupportedOperationException();
}
functionDetails = FunctionConfigUtils.convert(functionConfig, userCodeClassLoader != null ? userCodeClassLoader : Thread.currentThread().getContextClassLoader());
} else if (sourceConfig != null) {
inferMissingArguments(sourceConfig);
userCodeFile = sourceConfig.getArchive();
parallelism = sourceConfig.getParallelism();
ClassLoader builtInSourceClassLoader = userCodeFile != null ? isBuiltInSource(userCodeFile) : null;
if (builtInSourceClassLoader != null) {
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, builtInSourceClassLoader, true));
userCodeClassLoader = builtInSourceClassLoader;
} else if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
userCodeClassLoader = sourceClassLoader;
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("Source archive (" + userCodeFile + ") does not exist");
}
ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
userCodeClassLoader = sourceClassLoader;
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The archive property must be specified in SourceConfig.");
}
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, Thread.currentThread().getContextClassLoader(), true));
}
} else if (sinkConfig != null) {
inferMissingArguments(sinkConfig);
userCodeFile = sinkConfig.getArchive();
parallelism = sinkConfig.getParallelism();
ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null;
if (builtInSinkClassLoader != null) {
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, builtInSinkClassLoader, true));
userCodeClassLoader = builtInSinkClassLoader;
} else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
userCodeClassLoader = sinkClassLoader;
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("Sink archive does not exist");
}
ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
userCodeClassLoader = sinkClassLoader;
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The archive property must be specified in SourceConfig.");
}
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, Thread.currentThread().getContextClassLoader(), true));
}
} else {
throw new IllegalArgumentException("Must specify Function, Source or Sink config");
}
if (System.getProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) == null) {
System.setProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY, LocalRunner.class.getProtectionDomain().getCodeSource().getLocation().getFile());
}
AuthenticationConfig authConfig = AuthenticationConfig.builder().clientAuthenticationPlugin(clientAuthPlugin).clientAuthenticationParameters(clientAuthParams).useTls(useTls).tlsAllowInsecureConnection(tlsAllowInsecureConnection).tlsHostnameVerificationEnable(tlsHostNameVerificationEnabled).tlsTrustCertsFilePath(tlsTrustCertFilePath).build();
String serviceUrl = DEFAULT_SERVICE_URL;
if (brokerServiceUrl != null) {
serviceUrl = brokerServiceUrl;
}
if (webServiceUrl == null) {
webServiceUrl = DEFAULT_WEB_SERVICE_URL;
}
if ((sourceConfig != null || sinkConfig != null || functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) && (runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
// By default run java functions as threads
startThreadedMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
} else {
startProcessMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
}
local.addAll(spawners);
}
if (blocking) {
if (exitOnError) {
for (RuntimeSpawner spawner : local) {
spawner.join();
log.info("RuntimeSpawner quit because of", spawner.getRuntime().getDeathException());
}
close();
} else {
synchronized (this) {
while (running.get()) {
this.wait();
}
}
}
}
}
Also used :
Functions(org.apache.pulsar.functions.utils.functions.Functions)
LinkedList(java.util.LinkedList)
Function(org.apache.pulsar.functions.proto.Function)
AuthenticationConfig(org.apache.pulsar.functions.instance.AuthenticationConfig)
WorkerConfig(org.apache.pulsar.functions.worker.WorkerConfig)
File(java.io.File)
RuntimeSpawner(org.apache.pulsar.functions.runtime.RuntimeSpawner)
Example 4 with Functions
use of org.apache.pulsar.functions.utils.functions.Functions in project incubator-pulsar by apache.
the class LocalRunner method start.
public void start(boolean blocking) throws Exception {
List<RuntimeSpawner> local = new LinkedList<>();
synchronized (this) {
if (!running.compareAndSet(false, true)) {
throw new IllegalArgumentException("Pulsar Function local run already started!");
}
Runtime.getRuntime().addShutdownHook(shutdownHook);
Function.FunctionDetails functionDetails;
String userCodeFile = null;
int parallelism;
if (functionConfig != null) {
FunctionConfigUtils.inferMissingArguments(functionConfig, true);
parallelism = functionConfig.getParallelism();
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
userCodeFile = functionConfig.getJar();
boolean isBuiltin = !StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(Utils.BUILTIN);
if (isBuiltin) {
WorkerConfig workerConfig = WorkerConfig.load(System.getenv("PULSAR_HOME") + "/conf/functions_worker.yml");
Functions functions = FunctionUtils.searchForFunctions(System.getenv("PULSAR_HOME") + workerConfig.getFunctionsDirectory().replaceFirst("^.", ""));
String functionType = functionConfig.getJar().replaceFirst("^builtin://", "");
userCodeFile = functions.getFunctions().get(functionType).toString();
}
if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("User jar does not exist");
}
userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The jar property must be specified in FunctionConfig.");
}
FunctionConfigUtils.validateJavaFunction(functionConfig, Thread.currentThread().getContextClassLoader());
}
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
userCodeFile = functionConfig.getGo();
} else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) {
userCodeFile = functionConfig.getPy();
} else {
throw new UnsupportedOperationException();
}
functionDetails = FunctionConfigUtils.convert(functionConfig, userCodeClassLoader != null ? userCodeClassLoader : Thread.currentThread().getContextClassLoader());
} else if (sourceConfig != null) {
inferMissingArguments(sourceConfig);
userCodeFile = sourceConfig.getArchive();
parallelism = sourceConfig.getParallelism();
ClassLoader builtInSourceClassLoader = userCodeFile != null ? isBuiltInSource(userCodeFile) : null;
if (builtInSourceClassLoader != null) {
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, builtInSourceClassLoader, true));
userCodeClassLoader = builtInSourceClassLoader;
} else if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
userCodeClassLoader = sourceClassLoader;
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("Source archive (" + userCodeFile + ") does not exist");
}
ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
userCodeClassLoader = sourceClassLoader;
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The archive property must be specified in SourceConfig.");
}
functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, Thread.currentThread().getContextClassLoader(), true));
}
} else if (sinkConfig != null) {
inferMissingArguments(sinkConfig);
userCodeFile = sinkConfig.getArchive();
parallelism = sinkConfig.getParallelism();
ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null;
if (builtInSinkClassLoader != null) {
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, builtInSinkClassLoader, true));
userCodeClassLoader = builtInSinkClassLoader;
} else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
userCodeClassLoader = sinkClassLoader;
userCodeClassLoaderCreated = true;
} else if (userCodeFile != null) {
File file = new File(userCodeFile);
if (!file.exists()) {
throw new RuntimeException("Sink archive does not exist");
}
ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
userCodeClassLoader = sinkClassLoader;
userCodeClassLoaderCreated = true;
} else {
if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
throw new IllegalStateException("The archive property must be specified in SourceConfig.");
}
functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, Thread.currentThread().getContextClassLoader(), true));
}
} else {
throw new IllegalArgumentException("Must specify Function, Source or Sink config");
}
if (System.getProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) == null) {
System.setProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY, LocalRunner.class.getProtectionDomain().getCodeSource().getLocation().getFile());
}
AuthenticationConfig authConfig = AuthenticationConfig.builder().clientAuthenticationPlugin(clientAuthPlugin).clientAuthenticationParameters(clientAuthParams).useTls(useTls).tlsAllowInsecureConnection(tlsAllowInsecureConnection).tlsHostnameVerificationEnable(tlsHostNameVerificationEnabled).tlsTrustCertsFilePath(tlsTrustCertFilePath).build();
String serviceUrl = DEFAULT_SERVICE_URL;
if (brokerServiceUrl != null) {
serviceUrl = brokerServiceUrl;
}
if (webServiceUrl == null) {
webServiceUrl = DEFAULT_WEB_SERVICE_URL;
}
if ((sourceConfig != null || sinkConfig != null || functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) && (runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
// By default run java functions as threads
startThreadedMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
} else {
startProcessMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
}
local.addAll(spawners);
}
if (blocking) {
if (exitOnError) {
for (RuntimeSpawner spawner : local) {
spawner.join();
log.info("RuntimeSpawner quit because of", spawner.getRuntime().getDeathException());
}
close();
} else {
synchronized (this) {
while (running.get()) {
this.wait();
}
}
}
}
}
Also used :
Functions(org.apache.pulsar.functions.utils.functions.Functions)
LinkedList(java.util.LinkedList)
Function(org.apache.pulsar.functions.proto.Function)
AuthenticationConfig(org.apache.pulsar.functions.instance.AuthenticationConfig)
WorkerConfig(org.apache.pulsar.functions.worker.WorkerConfig)
File(java.io.File)
RuntimeSpawner(org.apache.pulsar.functions.runtime.RuntimeSpawner)
Example 5 with Functions
use of org.apache.pulsar.functions.utils.functions.Functions in project incubator-pulsar by apache.
the class ComponentImpl method getStreamingOutput.
private StreamingOutput getStreamingOutput(String pkgPath) {
final StreamingOutput streamingOutput = output -> {
if (pkgPath.startsWith(Utils.HTTP)) {
URL url = URI.create(pkgPath).toURL();
try (InputStream inputStream = url.openStream()) {
IOUtils.copy(inputStream, output);
}
} else if (pkgPath.startsWith(Utils.FILE)) {
URI url = URI.create(pkgPath);
File file = new File(url.getPath());
Files.copy(file.toPath(), output);
} else if (pkgPath.startsWith(Utils.BUILTIN) && !worker().getWorkerConfig().getUploadBuiltinSinksSources()) {
String sType = pkgPath.replaceFirst("^builtin://", "");
final String connectorsDir = worker().getWorkerConfig().getConnectorsDirectory();
log.warn("Processing package {} ; looking at the dir {}", pkgPath, connectorsDir);
Functions sinksOrSources = FunctionUtils.searchForFunctions(connectorsDir, true);
Path narPath = sinksOrSources.getFunctions().get(sType);
if (narPath == null) {
throw new IllegalStateException("Didn't find " + pkgPath + " in " + connectorsDir);
}
log.info("Loading {} from {}", pkgPath, narPath);
try (InputStream in = new FileInputStream(narPath.toString())) {
IOUtils.copy(in, output, 1024);
output.flush();
}
} else if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) {
try {
File file = downloadPackageFile(worker(), pkgPath);
try (InputStream in = new FileInputStream(file)) {
IOUtils.copy(in, output, 1024);
output.flush();
}
} catch (Exception e) {
log.error("Failed download package {} from packageMangment Service", pkgPath, e);
}
} else {
WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), output, pkgPath);
}
};
return streamingOutput;
}
Also used :
RestUtils.throwUnavailableException(org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException)
Functions(org.apache.pulsar.functions.utils.functions.Functions)
FormDataContentDisposition(org.glassfish.jersey.media.multipart.FormDataContentDisposition)
StorageClient(org.apache.bookkeeper.api.StorageClient)
URL(java.net.URL)
Producer(org.apache.pulsar.client.api.Producer)
SchemaSerializationException(org.apache.pulsar.client.api.SchemaSerializationException)
FunctionMetaData(org.apache.pulsar.functions.proto.Function.FunctionMetaData)
Unpooled(io.netty.buffer.Unpooled)
StorageAdminClient(org.apache.bookkeeper.clients.admin.StorageAdminClient)
FunctionCommon.getStateNamespace(org.apache.pulsar.functions.utils.FunctionCommon.getStateNamespace)
StorageClientSettings(org.apache.bookkeeper.clients.config.StorageClientSettings)
FunctionDetails(org.apache.pulsar.functions.proto.Function.FunctionDetails)
FunctionConfigUtils(org.apache.pulsar.functions.utils.FunctionConfigUtils)
FunctionCommon.isFunctionCodeBuiltin(org.apache.pulsar.functions.utils.FunctionCommon.isFunctionCodeBuiltin)
PulsarWorkerService(org.apache.pulsar.functions.worker.PulsarWorkerService)
FunctionConfig(org.apache.pulsar.common.functions.FunctionConfig)
NamespaceName(org.apache.pulsar.common.naming.NamespaceName)
UriBuilder(javax.ws.rs.core.UriBuilder)
KeyValue(org.apache.bookkeeper.api.kv.result.KeyValue)
URI(java.net.URI)
WorkerInfo(org.apache.pulsar.common.functions.WorkerInfo)
StringUtils.isEmpty(org.apache.commons.lang3.StringUtils.isEmpty)
Path(java.nio.file.Path)
FunctionMetaDataUtils(org.apache.pulsar.functions.utils.FunctionMetaDataUtils)
Function(org.apache.pulsar.functions.proto.Function)
FunctionCommon.createPkgTempFile(org.apache.pulsar.functions.utils.FunctionCommon.createPkgTempFile)
WorkerUtils(org.apache.pulsar.functions.worker.WorkerUtils)
Collection(java.util.Collection)
StreamingOutput(javax.ws.rs.core.StreamingOutput)
StorageClientBuilder(org.apache.bookkeeper.clients.StorageClientBuilder)
PackageMetadata(org.apache.pulsar.packages.management.core.common.PackageMetadata)
AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource)
IOUtils(org.apache.commons.io.IOUtils)
FunctionMetaDataManager(org.apache.pulsar.functions.worker.FunctionMetaDataManager)
Base64(java.util.Base64)
List(java.util.List)
Slf4j(lombok.extern.slf4j.Slf4j)
StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank)
Response(javax.ws.rs.core.Response)
WebApplicationException(javax.ws.rs.WebApplicationException)
Table(org.apache.bookkeeper.api.kv.Table)
TenantInfoImpl(org.apache.pulsar.common.policies.data.TenantInfoImpl)
PackageName(org.apache.pulsar.packages.management.core.common.PackageName)
Utils(org.apache.pulsar.common.functions.Utils)
SourceSpec(org.apache.pulsar.functions.proto.Function.SourceSpec)
TopicName(org.apache.pulsar.common.naming.TopicName)
RuntimeSpawner(org.apache.pulsar.functions.runtime.RuntimeSpawner)
ConnectorDefinition(org.apache.pulsar.common.io.ConnectorDefinition)
NamespaceNotFoundException(org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException)
InstanceCommunication(org.apache.pulsar.functions.proto.InstanceCommunication)
Message(org.apache.pulsar.client.api.Message)
AtomicReference(java.util.concurrent.atomic.AtomicReference)
Supplier(java.util.function.Supplier)
RestException(org.apache.pulsar.common.util.RestException)
FunctionsImpl.downloadPackageFile(org.apache.pulsar.functions.worker.rest.api.FunctionsImpl.downloadPackageFile)
ByteBuf(io.netty.buffer.ByteBuf)
FunctionInstanceStatsDataImpl(org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl)
FunctionCommon.getUniquePackageName(org.apache.pulsar.functions.utils.FunctionCommon.getUniquePackageName)
LinkedList(java.util.LinkedList)
FunctionsImpl(org.apache.pulsar.client.admin.internal.FunctionsImpl)
Status(javax.ws.rs.core.Response.Status)
FunctionState(org.apache.pulsar.common.functions.FunctionState)
ComponentTypeUtils(org.apache.pulsar.functions.utils.ComponentTypeUtils)
InstanceUtils(org.apache.pulsar.functions.instance.InstanceUtils)
StreamNotFoundException(org.apache.bookkeeper.clients.exceptions.StreamNotFoundException)
Files(java.nio.file.Files)
UTF_8(java.nio.charset.StandardCharsets.UTF_8)
PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException)
FunctionUtils(org.apache.pulsar.functions.utils.functions.FunctionUtils)
Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull)
PackageLocationMetaData(org.apache.pulsar.functions.proto.Function.PackageLocationMetaData)
FileOutputStream(java.io.FileOutputStream)
Reader(org.apache.pulsar.client.api.Reader)
FunctionRuntimeInfo(org.apache.pulsar.functions.worker.FunctionRuntimeInfo)
IOException(java.io.IOException)
FileInputStream(java.io.FileInputStream)
File(java.io.File)
Schema(org.apache.pulsar.client.api.Schema)
ExecutionException(java.util.concurrent.ExecutionException)
TimeUnit(java.util.concurrent.TimeUnit)
ByteBufUtil(io.netty.buffer.ByteBufUtil)
MessageId(org.apache.pulsar.client.api.MessageId)
FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon)
FutureUtils.result(org.apache.bookkeeper.common.concurrent.FutureUtils.result)
FunctionStatsImpl(org.apache.pulsar.common.policies.data.FunctionStatsImpl)
Codec(org.apache.pulsar.common.util.Codec)
WorkerService(org.apache.pulsar.functions.worker.WorkerService)
SinkSpec(org.apache.pulsar.functions.proto.Function.SinkSpec)
FunctionRuntimeManager(org.apache.pulsar.functions.worker.FunctionRuntimeManager)
SECONDS(java.util.concurrent.TimeUnit.SECONDS)
InputStream(java.io.InputStream)
Component(org.apache.pulsar.functions.worker.service.api.Component)
Path(java.nio.file.Path)
FileInputStream(java.io.FileInputStream)
InputStream(java.io.InputStream)
StreamingOutput(javax.ws.rs.core.StreamingOutput)
Functions(org.apache.pulsar.functions.utils.functions.Functions)
URI(java.net.URI)
FunctionCommon.createPkgTempFile(org.apache.pulsar.functions.utils.FunctionCommon.createPkgTempFile)
FunctionsImpl.downloadPackageFile(org.apache.pulsar.functions.worker.rest.api.FunctionsImpl.downloadPackageFile)
File(java.io.File)
URL(java.net.URL)
FileInputStream(java.io.FileInputStream)
RestUtils.throwUnavailableException(org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException)
SchemaSerializationException(org.apache.pulsar.client.api.SchemaSerializationException)
WebApplicationException(javax.ws.rs.WebApplicationException)
NamespaceNotFoundException(org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException)
RestException(org.apache.pulsar.common.util.RestException)
StreamNotFoundException(org.apache.bookkeeper.clients.exceptions.StreamNotFoundException)
PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException)
IOException(java.io.IOException)
ExecutionException(java.util.concurrent.ExecutionException)
Aggregations
File (java.io.File)6
LinkedList (java.util.LinkedList)6
Function (org.apache.pulsar.functions.proto.Function)4
RuntimeSpawner (org.apache.pulsar.functions.runtime.RuntimeSpawner)4
Functions (org.apache.pulsar.functions.utils.functions.Functions)4
Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)3
ByteBuf (io.netty.buffer.ByteBuf)3
ByteBufUtil (io.netty.buffer.ByteBufUtil)3
Unpooled (io.netty.buffer.Unpooled)3
FileInputStream (java.io.FileInputStream)3
IOException (java.io.IOException)3
InputStream (java.io.InputStream)3
URI (java.net.URI)3
URL (java.net.URL)3
UTF_8 (java.nio.charset.StandardCharsets.UTF_8)3
Files (java.nio.file.Files)3
Path (java.nio.file.Path)3
Base64 (java.util.Base64)3
Collection (java.util.Collection)3
List (java.util.List)3
