Examples with SecretsProviderConfigurator org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator used on opensource projects

Example 1 with SecretsProviderConfigurator

use of org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator in project pulsar by yahoo.

the class KubernetesRuntimeFactory method createContainer.

@Override
public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String codePkgUrl, String originalCodeFileName, Long expectedHealthCheckInterval) throws Exception {
    String instanceFile = null;
    switch(instanceConfig.getFunctionDetails().getRuntime()) {
        case JAVA:
            instanceFile = javaInstanceJarFile;
            break;
        case PYTHON:
            instanceFile = pythonInstanceFile;
            break;
        case GO:
            break;
        default:
            throw new RuntimeException("Unsupported Runtime " + instanceConfig.getFunctionDetails().getRuntime());
    }
    // adjust the auth config to support auth
    if (authenticationEnabled) {
        authProvider.ifPresent(kubernetesFunctionAuthProvider -> kubernetesFunctionAuthProvider.configureAuthenticationConfig(authConfig, Optional.ofNullable(getFunctionAuthData(Optional.ofNullable(instanceConfig.getFunctionAuthenticationSpec())))));
    }
    Optional<KubernetesManifestCustomizer> manifestCustomizer = getRuntimeCustomizer();
    String overriddenNamespace = manifestCustomizer.map((customizer) -> customizer.customizeNamespace(instanceConfig.getFunctionDetails(), jobNamespace)).orElse(jobNamespace);
    String overriddenName = manifestCustomizer.map((customizer) -> customizer.customizeName(instanceConfig.getFunctionDetails(), jobName)).orElse(jobName);
    // pass metricsPort configured in functionRuntimeFactoryConfigs.metricsPort in functions_worker.yml
    if (metricsPort != null) {
        instanceConfig.setMetricsPort(metricsPort);
    }
    return new KubernetesRuntime(appsClient, coreClient, // get the namespace for this function
    overriddenNamespace, overriddenName, customLabels, installUserCodeDependencies, pythonDependencyRepository, pythonExtraDependencyRepository, pulsarDockerImageName, functionDockerImages, imagePullPolicy, pulsarRootDir, instanceConfig, instanceFile, extraDependenciesDir, logDirectory, configAdminCLI, codePkgUrl, originalCodeFileName, pulsarServiceUrl, pulsarAdminUrl, stateStorageServiceUri, authConfig, secretsProviderConfigurator, expectedMetricsCollectionInterval, percentMemoryPadding, cpuOverCommitRatio, memoryOverCommitRatio, gracePeriodSeconds, authProvider, authenticationEnabled, grpcPort, narExtractionDirectory, manifestCustomizer, functionInstanceClassPath, downloadDirectory);
}

Example 2 with SecretsProviderConfigurator

use of org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator in project pulsar by yahoo.

the class LocalRunner method startProcessMode.

private void startProcessMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails, int parallelism, int instanceIdOffset, String serviceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, String userCodeFile) throws Exception {
    SecretsProviderConfigurator secretsProviderConfigurator = getSecretsProviderConfigurator();
    runtimeFactory = new ProcessRuntimeFactory(serviceUrl, webServiceUrl, stateStorageServiceUrl, authConfig, null, /* java instance jar file */
    null, /* python instance file */
    null, /* log directory */
    null, /* extra dependencies dir */
    narExtractionDirectory, /* nar extraction dir */
    secretsProviderConfigurator, false, Optional.empty(), Optional.empty());
    for (int i = 0; i < parallelism; ++i) {
        InstanceConfig instanceConfig = new InstanceConfig();
        instanceConfig.setFunctionDetails(functionDetails);
        // TODO: correctly implement function version and id
        instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
        instanceConfig.setFunctionId(UUID.randomUUID().toString());
        instanceConfig.setInstanceId(i + instanceIdOffset);
        instanceConfig.setMaxBufferedTuples(1024);
        instanceConfig.setPort(FunctionCommon.findAvailablePort());
        if (metricsPortStart != null) {
            int metricsPort = metricsPortStart + i;
            if (metricsPortStart < 0 || metricsPortStart > 65535) {
                throw new IllegalArgumentException("Metrics port need to be within the range of 0 and 65535");
            }
            instanceConfig.setMetricsPort(metricsPort);
        } else {
            instanceConfig.setMetricsPort(FunctionCommon.findAvailablePort());
        }
        instanceConfig.setClusterName("local");
        if (functionConfig != null) {
            instanceConfig.setMaxPendingAsyncRequests(functionConfig.getMaxPendingAsyncRequests());
            if (functionConfig.getExposePulsarAdminClientEnabled() != null) {
                instanceConfig.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
            }
        }
        RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, userCodeFile, null, runtimeFactory, instanceLivenessCheck);
        spawners.add(runtimeSpawner);
        runtimeSpawner.start();
    }
    Timer statusCheckTimer = new Timer();
    statusCheckTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            CompletableFuture<String>[] futures = new CompletableFuture[spawners.size()];
            int index = 0;
            for (RuntimeSpawner spawner : spawners) {
                futures[index] = spawner.getFunctionStatusAsJson(index);
                index++;
            }
            try {
                CompletableFuture.allOf(futures).get(5, TimeUnit.SECONDS);
                for (index = 0; index < futures.length; ++index) {
                    String json = futures[index].get();
                    Gson gson = new GsonBuilder().setPrettyPrinting().create();
                    log.info(gson.toJson(JsonParser.parseString(json)));
                }
            } catch (TimeoutException | InterruptedException | ExecutionException e) {
                log.error("Could not get status from all local instances");
            }
        }
    }, 30000, 30000);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread(() -> statusCheckTimer.cancel()));
}

Example 3 with SecretsProviderConfigurator

use of org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator in project incubator-pulsar by apache.

the class KubernetesRuntimeFactory method createContainer.

@Override
public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String codePkgUrl, String originalCodeFileName, Long expectedHealthCheckInterval) throws Exception {
    String instanceFile = null;
    switch(instanceConfig.getFunctionDetails().getRuntime()) {
        case JAVA:
            instanceFile = javaInstanceJarFile;
            break;
        case PYTHON:
            instanceFile = pythonInstanceFile;
            break;
        case GO:
            break;
        default:
            throw new RuntimeException("Unsupported Runtime " + instanceConfig.getFunctionDetails().getRuntime());
    }
    // adjust the auth config to support auth
    if (authenticationEnabled) {
        authProvider.ifPresent(kubernetesFunctionAuthProvider -> kubernetesFunctionAuthProvider.configureAuthenticationConfig(authConfig, Optional.ofNullable(getFunctionAuthData(Optional.ofNullable(instanceConfig.getFunctionAuthenticationSpec())))));
    }
    Optional<KubernetesManifestCustomizer> manifestCustomizer = getRuntimeCustomizer();
    String overriddenNamespace = manifestCustomizer.map((customizer) -> customizer.customizeNamespace(instanceConfig.getFunctionDetails(), jobNamespace)).orElse(jobNamespace);
    String overriddenName = manifestCustomizer.map((customizer) -> customizer.customizeName(instanceConfig.getFunctionDetails(), jobName)).orElse(jobName);
    // pass metricsPort configured in functionRuntimeFactoryConfigs.metricsPort in functions_worker.yml
    if (metricsPort != null) {
        instanceConfig.setMetricsPort(metricsPort);
    }
    return new KubernetesRuntime(appsClient, coreClient, // get the namespace for this function
    overriddenNamespace, overriddenName, customLabels, installUserCodeDependencies, pythonDependencyRepository, pythonExtraDependencyRepository, pulsarDockerImageName, functionDockerImages, imagePullPolicy, pulsarRootDir, instanceConfig, instanceFile, extraDependenciesDir, logDirectory, configAdminCLI, codePkgUrl, originalCodeFileName, pulsarServiceUrl, pulsarAdminUrl, stateStorageServiceUri, authConfig, secretsProviderConfigurator, expectedMetricsCollectionInterval, percentMemoryPadding, cpuOverCommitRatio, memoryOverCommitRatio, gracePeriodSeconds, authProvider, authenticationEnabled, grpcPort, narExtractionDirectory, manifestCustomizer, functionInstanceClassPath, downloadDirectory);
}

Example 4 with SecretsProviderConfigurator

use of org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator in project incubator-pulsar by apache.

the class LocalRunner method startProcessMode.

private void startProcessMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails, int parallelism, int instanceIdOffset, String serviceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, String userCodeFile) throws Exception {
    SecretsProviderConfigurator secretsProviderConfigurator = getSecretsProviderConfigurator();
    runtimeFactory = new ProcessRuntimeFactory(serviceUrl, webServiceUrl, stateStorageServiceUrl, authConfig, null, /* java instance jar file */
    null, /* python instance file */
    null, /* log directory */
    null, /* extra dependencies dir */
    narExtractionDirectory, /* nar extraction dir */
    secretsProviderConfigurator, false, Optional.empty(), Optional.empty());
    for (int i = 0; i < parallelism; ++i) {
        InstanceConfig instanceConfig = new InstanceConfig();
        instanceConfig.setFunctionDetails(functionDetails);
        // TODO: correctly implement function version and id
        instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
        instanceConfig.setFunctionId(UUID.randomUUID().toString());
        instanceConfig.setInstanceId(i + instanceIdOffset);
        instanceConfig.setMaxBufferedTuples(1024);
        instanceConfig.setPort(FunctionCommon.findAvailablePort());
        if (metricsPortStart != null) {
            int metricsPort = metricsPortStart + i;
            if (metricsPortStart < 0 || metricsPortStart > 65535) {
                throw new IllegalArgumentException("Metrics port need to be within the range of 0 and 65535");
            }
            instanceConfig.setMetricsPort(metricsPort);
        } else {
            instanceConfig.setMetricsPort(FunctionCommon.findAvailablePort());
        }
        instanceConfig.setClusterName("local");
        if (functionConfig != null) {
            instanceConfig.setMaxPendingAsyncRequests(functionConfig.getMaxPendingAsyncRequests());
            if (functionConfig.getExposePulsarAdminClientEnabled() != null) {
                instanceConfig.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
            }
        }
        RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, userCodeFile, null, runtimeFactory, instanceLivenessCheck);
        spawners.add(runtimeSpawner);
        runtimeSpawner.start();
    }
    Timer statusCheckTimer = new Timer();
    statusCheckTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            CompletableFuture<String>[] futures = new CompletableFuture[spawners.size()];
            int index = 0;
            for (RuntimeSpawner spawner : spawners) {
                futures[index] = spawner.getFunctionStatusAsJson(index);
                index++;
            }
            try {
                CompletableFuture.allOf(futures).get(5, TimeUnit.SECONDS);
                for (index = 0; index < futures.length; ++index) {
                    String json = futures[index].get();
                    Gson gson = new GsonBuilder().setPrettyPrinting().create();
                    log.info(gson.toJson(JsonParser.parseString(json)));
                }
            } catch (TimeoutException | InterruptedException | ExecutionException e) {
                log.error("Could not get status from all local instances");
            }
        }
    }, 30000, 30000);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread(() -> statusCheckTimer.cancel()));
}

Example 5 with SecretsProviderConfigurator

use of org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator in project incubator-pulsar by apache.

the class LocalRunner method getSecretsProviderConfigurator.

private SecretsProviderConfigurator getSecretsProviderConfigurator() {
    SecretsProviderConfigurator secretsProviderConfigurator;
    if (secretsProviderClassName != null) {
        Map<String, String> config = null;
        if (secretsProviderConfig != null) {
            config = (Map<String, String>) new Gson().fromJson(secretsProviderConfig, Map.class);
        }
        secretsProviderConfigurator = new NameAndConfigBasedSecretsProviderConfigurator(secretsProviderClassName, config);
    } else {
        secretsProviderConfigurator = new DefaultSecretsProviderConfigurator();
    }
    return secretsProviderConfigurator;
}