Examples with AuthenticationConfig org.apache.pulsar.functions.instance.AuthenticationConfig used on opensource projects

Example 1 with AuthenticationConfig

use of org.apache.pulsar.functions.instance.AuthenticationConfig in project pulsar by apache.

the class KubernetesRuntimeFactoryTest method getKuberentesRuntimeFactory.

private KubernetesRuntimeFactory getKuberentesRuntimeFactory() {
    KubernetesRuntimeFactory kubernetesRuntimeFactory = new KubernetesRuntimeFactory();
    WorkerConfig workerConfig = new WorkerConfig();
    Map<String, String> imageNames = new HashMap<>();
    imageNames.put("JAVA", "test-java-function-docker-image");
    imageNames.put("PYTHON", "test-python-function-docker-image");
    imageNames.put("GO", "test-go-function-docker-image");
    KubernetesRuntimeFactoryConfig kubernetesRuntimeFactoryConfig = new KubernetesRuntimeFactoryConfig();
    kubernetesRuntimeFactoryConfig.setK8Uri("test_k8uri");
    kubernetesRuntimeFactoryConfig.setJobNamespace("test_jobNamespace");
    kubernetesRuntimeFactoryConfig.setJobName("test_jobName");
    kubernetesRuntimeFactoryConfig.setPulsarDockerImageName("test_dockerImage");
    kubernetesRuntimeFactoryConfig.setFunctionDockerImages(imageNames);
    kubernetesRuntimeFactoryConfig.setImagePullPolicy("test_imagePullPolicy");
    workerConfig.setFunctionRuntimeFactoryClassName(KubernetesRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(kubernetesRuntimeFactoryConfig, Map.class));
    AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build();
    kubernetesRuntimeFactory.initialize(workerConfig, authenticationConfig, new DefaultSecretsProviderConfigurator(), Mockito.mock(ConnectorsManager.class), Optional.empty(), Optional.empty());
    return kubernetesRuntimeFactory;
}

Example 2 with AuthenticationConfig

use of org.apache.pulsar.functions.instance.AuthenticationConfig in project pulsar by apache.

the class KubernetesRuntimeTest method assertMetricsPortConfigured.

private void assertMetricsPortConfigured(Map<String, Object> functionRuntimeFactoryConfigs, int expectedPort) throws Exception {
    KubernetesRuntimeFactory kubernetesRuntimeFactory = new KubernetesRuntimeFactory();
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setFunctionRuntimeFactoryClassName(KubernetesRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(functionRuntimeFactoryConfigs);
    AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build();
    kubernetesRuntimeFactory.initialize(workerConfig, authenticationConfig, new DefaultSecretsProviderConfigurator(), Mockito.mock(ConnectorsManager.class), Optional.empty(), Optional.empty());
    InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true);
    KubernetesRuntime container = kubernetesRuntimeFactory.createContainer(config, userJarFile, userJarFile, 30l);
    V1PodTemplateSpec template = container.createStatefulSet().getSpec().getTemplate();
    Map<String, String> annotations = template.getMetadata().getAnnotations();
    if (expectedPort != -1) {
        // metrics port should be passed to k8s annotation for prometheus scraping
        assertEquals(annotations.get("prometheus.io/port"), String.valueOf(expectedPort));
        // scraping annotation should exist
        assertEquals(annotations.get("prometheus.io/scrape"), "true");
        // metrics port should be passed to JavaInstanceStarter with --metrics_port argument
        assertTrue(container.getProcessArgs().stream().collect(Collectors.joining(" ")).contains("--metrics_port " + expectedPort));
    } else {
        // No prometheus annotations should exist
        assertFalse(annotations.containsKey("prometheus.io/scrape"));
        assertFalse(annotations.containsKey("prometheus.io/port"));
        // metrics will be started on random port when the port isn't specified
        // check that "--metrics_port 0" argument is passed
        assertTrue(container.getProcessArgs().stream().collect(Collectors.joining(" ")).contains("--metrics_port 0"));
    }
}

Example 3 with AuthenticationConfig

use of org.apache.pulsar.functions.instance.AuthenticationConfig in project pulsar by apache.

the class KubernetesSecretsTokenAuthProviderTest method configureAuthenticationConfigNoCa.

@Test
public void configureAuthenticationConfigNoCa() {
    CoreV1Api coreV1Api = mock(CoreV1Api.class);
    KubernetesSecretsTokenAuthProvider kubernetesSecretsTokenAuthProvider = new KubernetesSecretsTokenAuthProvider();
    kubernetesSecretsTokenAuthProvider.initialize(coreV1Api, null, (fd) -> "default");
    AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build();
    FunctionAuthData functionAuthData = FunctionAuthData.builder().data("foo".getBytes()).build();
    kubernetesSecretsTokenAuthProvider.configureAuthenticationConfig(authenticationConfig, Optional.of(functionAuthData));
    Assert.assertEquals(authenticationConfig.getClientAuthenticationPlugin(), AuthenticationToken.class.getName());
    Assert.assertEquals(authenticationConfig.getClientAuthenticationParameters(), "file:///etc/auth/token");
    Assert.assertEquals(authenticationConfig.getTlsTrustCertsFilePath(), null);
}

Example 4 with AuthenticationConfig

use of org.apache.pulsar.functions.instance.AuthenticationConfig in project pulsar by apache.

the class LocalRunner method start.

public void start(boolean blocking) throws Exception {
    List<RuntimeSpawner> local = new LinkedList<>();
    synchronized (this) {
        if (!running.compareAndSet(false, true)) {
            throw new IllegalArgumentException("Pulsar Function local run already started!");
        }
        Runtime.getRuntime().addShutdownHook(shutdownHook);
        Function.FunctionDetails functionDetails;
        String userCodeFile = null;
        int parallelism;
        if (functionConfig != null) {
            FunctionConfigUtils.inferMissingArguments(functionConfig, true);
            parallelism = functionConfig.getParallelism();
            if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
                userCodeFile = functionConfig.getJar();
                boolean isBuiltin = !StringUtils.isEmpty(functionConfig.getJar()) && functionConfig.getJar().startsWith(Utils.BUILTIN);
                if (isBuiltin) {
                    WorkerConfig workerConfig = WorkerConfig.load(System.getenv("PULSAR_HOME") + "/conf/functions_worker.yml");
                    Functions functions = FunctionUtils.searchForFunctions(System.getenv("PULSAR_HOME") + workerConfig.getFunctionsDirectory().replaceFirst("^.", ""));
                    String functionType = functionConfig.getJar().replaceFirst("^builtin://", "");
                    userCodeFile = functions.getFunctions().get(functionType).toString();
                }
                if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
                    File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
                    userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
                    userCodeClassLoaderCreated = true;
                } else if (userCodeFile != null) {
                    File file = new File(userCodeFile);
                    if (!file.exists()) {
                        throw new RuntimeException("User jar does not exist");
                    }
                    userCodeClassLoader = FunctionConfigUtils.validate(functionConfig, file);
                    userCodeClassLoaderCreated = true;
                } else {
                    if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
                        throw new IllegalStateException("The jar property must be specified in FunctionConfig.");
                    }
                    FunctionConfigUtils.validateJavaFunction(functionConfig, Thread.currentThread().getContextClassLoader());
                }
            } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) {
                userCodeFile = functionConfig.getGo();
            } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) {
                userCodeFile = functionConfig.getPy();
            } else {
                throw new UnsupportedOperationException();
            }
            functionDetails = FunctionConfigUtils.convert(functionConfig, userCodeClassLoader != null ? userCodeClassLoader : Thread.currentThread().getContextClassLoader());
        } else if (sourceConfig != null) {
            inferMissingArguments(sourceConfig);
            userCodeFile = sourceConfig.getArchive();
            parallelism = sourceConfig.getParallelism();
            ClassLoader builtInSourceClassLoader = userCodeFile != null ? isBuiltInSource(userCodeFile) : null;
            if (builtInSourceClassLoader != null) {
                functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, builtInSourceClassLoader, true));
                userCodeClassLoader = builtInSourceClassLoader;
            } else if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) {
                File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
                ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
                functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
                userCodeClassLoader = sourceClassLoader;
                userCodeClassLoaderCreated = true;
            } else if (userCodeFile != null) {
                File file = new File(userCodeFile);
                if (!file.exists()) {
                    throw new RuntimeException("Source archive (" + userCodeFile + ") does not exist");
                }
                ClassLoader sourceClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SOURCE, sourceConfig.getClassName(), file, narExtractionDirectory);
                functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, sourceClassLoader, true));
                userCodeClassLoader = sourceClassLoader;
                userCodeClassLoaderCreated = true;
            } else {
                if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
                    throw new IllegalStateException("The archive property must be specified in SourceConfig.");
                }
                functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validateAndExtractDetails(sourceConfig, Thread.currentThread().getContextClassLoader(), true));
            }
        } else if (sinkConfig != null) {
            inferMissingArguments(sinkConfig);
            userCodeFile = sinkConfig.getArchive();
            parallelism = sinkConfig.getParallelism();
            ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null;
            if (builtInSinkClassLoader != null) {
                functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, builtInSinkClassLoader, true));
                userCodeClassLoader = builtInSinkClassLoader;
            } else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) {
                File file = FunctionCommon.extractFileFromPkgURL(userCodeFile);
                ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
                functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
                userCodeClassLoader = sinkClassLoader;
                userCodeClassLoaderCreated = true;
            } else if (userCodeFile != null) {
                File file = new File(userCodeFile);
                if (!file.exists()) {
                    throw new RuntimeException("Sink archive does not exist");
                }
                ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage(Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory);
                functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true));
                userCodeClassLoader = sinkClassLoader;
                userCodeClassLoaderCreated = true;
            } else {
                if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
                    throw new IllegalStateException("The archive property must be specified in SourceConfig.");
                }
                functionDetails = SinkConfigUtils.convert(sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, Thread.currentThread().getContextClassLoader(), true));
            }
        } else {
            throw new IllegalArgumentException("Must specify Function, Source or Sink config");
        }
        if (System.getProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY) == null) {
            System.setProperty(FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY, LocalRunner.class.getProtectionDomain().getCodeSource().getLocation().getFile());
        }
        AuthenticationConfig authConfig = AuthenticationConfig.builder().clientAuthenticationPlugin(clientAuthPlugin).clientAuthenticationParameters(clientAuthParams).useTls(useTls).tlsAllowInsecureConnection(tlsAllowInsecureConnection).tlsHostnameVerificationEnable(tlsHostNameVerificationEnabled).tlsTrustCertsFilePath(tlsTrustCertFilePath).build();
        String serviceUrl = DEFAULT_SERVICE_URL;
        if (brokerServiceUrl != null) {
            serviceUrl = brokerServiceUrl;
        }
        if (webServiceUrl == null) {
            webServiceUrl = DEFAULT_WEB_SERVICE_URL;
        }
        if ((sourceConfig != null || sinkConfig != null || functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) && (runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) {
            // By default run java functions as threads
            startThreadedMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
        } else {
            startProcessMode(functionDetails, parallelism, instanceIdOffset, serviceUrl, stateStorageServiceUrl, authConfig, userCodeFile);
        }
        local.addAll(spawners);
    }
    if (blocking) {
        if (exitOnError) {
            for (RuntimeSpawner spawner : local) {
                spawner.join();
                log.info("RuntimeSpawner quit because of", spawner.getRuntime().getDeathException());
            }
            close();
        } else {
            synchronized (this) {
                while (running.get()) {
                    this.wait();
                }
            }
        }
    }
}

Example 5 with AuthenticationConfig

use of org.apache.pulsar.functions.instance.AuthenticationConfig in project pulsar by yahoo.

the class KubernetesSecretsTokenAuthProviderTest method configureAuthenticationConfigNoCa.

@Test
public void configureAuthenticationConfigNoCa() {
    CoreV1Api coreV1Api = mock(CoreV1Api.class);
    KubernetesSecretsTokenAuthProvider kubernetesSecretsTokenAuthProvider = new KubernetesSecretsTokenAuthProvider();
    kubernetesSecretsTokenAuthProvider.initialize(coreV1Api, null, (fd) -> "default");
    AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build();
    FunctionAuthData functionAuthData = FunctionAuthData.builder().data("foo".getBytes()).build();
    kubernetesSecretsTokenAuthProvider.configureAuthenticationConfig(authenticationConfig, Optional.of(functionAuthData));
    Assert.assertEquals(authenticationConfig.getClientAuthenticationPlugin(), AuthenticationToken.class.getName());
    Assert.assertEquals(authenticationConfig.getClientAuthenticationParameters(), "file:///etc/auth/token");
    Assert.assertEquals(authenticationConfig.getTlsTrustCertsFilePath(), null);
}