Examples with ProcessRuntimeFactory org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory used on opensource projects

Example 1 with ProcessRuntimeFactory

use of org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory in project pulsar by yahoo.

the class FunctionRuntimeManagerTest method testFunctionRuntimeFactoryConfigsBackwardsCompatibility.

@Test
public void testFunctionRuntimeFactoryConfigsBackwardsCompatibility() throws Exception {
    // Test kubernetes runtime
    WorkerConfig.KubernetesContainerFactory kubernetesContainerFactory = new WorkerConfig.KubernetesContainerFactory();
    kubernetesContainerFactory.setK8Uri("k8Uri");
    kubernetesContainerFactory.setJobNamespace("jobNamespace");
    kubernetesContainerFactory.setJobName("jobName");
    kubernetesContainerFactory.setPulsarDockerImageName("pulsarDockerImageName");
    kubernetesContainerFactory.setImagePullPolicy("imagePullPolicy");
    kubernetesContainerFactory.setPulsarRootDir("pulsarRootDir");
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setKubernetesContainerFactory(kubernetesContainerFactory);
    try (MockedConstruction<KubernetesRuntimeFactory> mocked = Mockito.mockConstruction(KubernetesRuntimeFactory.class, withSettings().defaultAnswer(CALLS_REAL_METHODS), (mockedKubernetesRuntimeFactory, context) -> {
        doNothing().when(mockedKubernetesRuntimeFactory).initialize(any(WorkerConfig.class), any(AuthenticationConfig.class), any(SecretsProviderConfigurator.class), any(), any(), any());
        doNothing().when(mockedKubernetesRuntimeFactory).setupClient();
        doReturn(true).when(mockedKubernetesRuntimeFactory).externallyManaged();
    })) {
        FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager(workerConfig, mock(PulsarWorkerService.class), mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        KubernetesRuntimeFactory kubernetesRuntimeFactory = (KubernetesRuntimeFactory) functionRuntimeManager.getRuntimeFactory();
        assertEquals(kubernetesRuntimeFactory.getK8Uri(), "k8Uri");
        assertEquals(kubernetesRuntimeFactory.getJobNamespace(), "jobNamespace");
        assertEquals(kubernetesRuntimeFactory.getPulsarDockerImageName(), "pulsarDockerImageName");
        assertEquals(kubernetesRuntimeFactory.getImagePullPolicy(), "imagePullPolicy");
        assertEquals(kubernetesRuntimeFactory.getPulsarRootDir(), "pulsarRootDir");
        // Test process runtime
        WorkerConfig.ProcessContainerFactory processContainerFactory = new WorkerConfig.ProcessContainerFactory();
        processContainerFactory.setExtraFunctionDependenciesDir("extraDependenciesDir");
        processContainerFactory.setLogDirectory("logDirectory");
        processContainerFactory.setPythonInstanceLocation("pythonInstanceLocation");
        processContainerFactory.setJavaInstanceJarLocation("javaInstanceJarLocation");
        workerConfig = new WorkerConfig();
        workerConfig.setProcessContainerFactory(processContainerFactory);
        functionRuntimeManager = new FunctionRuntimeManager(workerConfig, mock(PulsarWorkerService.class), mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        assertEquals(functionRuntimeManager.getRuntimeFactory().getClass(), ProcessRuntimeFactory.class);
        ProcessRuntimeFactory processRuntimeFactory = (ProcessRuntimeFactory) functionRuntimeManager.getRuntimeFactory();
        assertEquals(processRuntimeFactory.getExtraDependenciesDir(), "extraDependenciesDir");
        assertEquals(processRuntimeFactory.getLogDirectory(), "logDirectory/functions");
        assertEquals(processRuntimeFactory.getPythonInstanceFile(), "pythonInstanceLocation");
        assertEquals(processRuntimeFactory.getJavaInstanceJarFile(), "javaInstanceJarLocation");
        // Test thread runtime
        WorkerConfig.ThreadContainerFactory threadContainerFactory = new WorkerConfig.ThreadContainerFactory();
        threadContainerFactory.setThreadGroupName("threadGroupName");
        workerConfig = new WorkerConfig();
        workerConfig.setThreadContainerFactory(threadContainerFactory);
        workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
        functionRuntimeManager = new FunctionRuntimeManager(workerConfig, mock(PulsarWorkerService.class), mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        assertEquals(functionRuntimeManager.getRuntimeFactory().getClass(), ThreadRuntimeFactory.class);
        ThreadRuntimeFactory threadRuntimeFactory = (ThreadRuntimeFactory) functionRuntimeManager.getRuntimeFactory();
        assertEquals(threadRuntimeFactory.getThreadGroup().getName(), "threadGroupName");
    }
}

Example 2 with ProcessRuntimeFactory

use of org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory in project pulsar by yahoo.

the class LocalRunner method startProcessMode.

private void startProcessMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails, int parallelism, int instanceIdOffset, String serviceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, String userCodeFile) throws Exception {
    SecretsProviderConfigurator secretsProviderConfigurator = getSecretsProviderConfigurator();
    runtimeFactory = new ProcessRuntimeFactory(serviceUrl, webServiceUrl, stateStorageServiceUrl, authConfig, null, /* java instance jar file */
    null, /* python instance file */
    null, /* log directory */
    null, /* extra dependencies dir */
    narExtractionDirectory, /* nar extraction dir */
    secretsProviderConfigurator, false, Optional.empty(), Optional.empty());
    for (int i = 0; i < parallelism; ++i) {
        InstanceConfig instanceConfig = new InstanceConfig();
        instanceConfig.setFunctionDetails(functionDetails);
        // TODO: correctly implement function version and id
        instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
        instanceConfig.setFunctionId(UUID.randomUUID().toString());
        instanceConfig.setInstanceId(i + instanceIdOffset);
        instanceConfig.setMaxBufferedTuples(1024);
        instanceConfig.setPort(FunctionCommon.findAvailablePort());
        if (metricsPortStart != null) {
            int metricsPort = metricsPortStart + i;
            if (metricsPortStart < 0 || metricsPortStart > 65535) {
                throw new IllegalArgumentException("Metrics port need to be within the range of 0 and 65535");
            }
            instanceConfig.setMetricsPort(metricsPort);
        } else {
            instanceConfig.setMetricsPort(FunctionCommon.findAvailablePort());
        }
        instanceConfig.setClusterName("local");
        if (functionConfig != null) {
            instanceConfig.setMaxPendingAsyncRequests(functionConfig.getMaxPendingAsyncRequests());
            if (functionConfig.getExposePulsarAdminClientEnabled() != null) {
                instanceConfig.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
            }
        }
        RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, userCodeFile, null, runtimeFactory, instanceLivenessCheck);
        spawners.add(runtimeSpawner);
        runtimeSpawner.start();
    }
    Timer statusCheckTimer = new Timer();
    statusCheckTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            CompletableFuture<String>[] futures = new CompletableFuture[spawners.size()];
            int index = 0;
            for (RuntimeSpawner spawner : spawners) {
                futures[index] = spawner.getFunctionStatusAsJson(index);
                index++;
            }
            try {
                CompletableFuture.allOf(futures).get(5, TimeUnit.SECONDS);
                for (index = 0; index < futures.length; ++index) {
                    String json = futures[index].get();
                    Gson gson = new GsonBuilder().setPrettyPrinting().create();
                    log.info(gson.toJson(JsonParser.parseString(json)));
                }
            } catch (TimeoutException | InterruptedException | ExecutionException e) {
                log.error("Could not get status from all local instances");
            }
        }
    }, 30000, 30000);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread(() -> statusCheckTimer.cancel()));
}

Example 3 with ProcessRuntimeFactory

use of org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory in project incubator-pulsar by apache.

the class LocalRunner method startProcessMode.

private void startProcessMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails, int parallelism, int instanceIdOffset, String serviceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, String userCodeFile) throws Exception {
    SecretsProviderConfigurator secretsProviderConfigurator = getSecretsProviderConfigurator();
    runtimeFactory = new ProcessRuntimeFactory(serviceUrl, webServiceUrl, stateStorageServiceUrl, authConfig, null, /* java instance jar file */
    null, /* python instance file */
    null, /* log directory */
    null, /* extra dependencies dir */
    narExtractionDirectory, /* nar extraction dir */
    secretsProviderConfigurator, false, Optional.empty(), Optional.empty());
    for (int i = 0; i < parallelism; ++i) {
        InstanceConfig instanceConfig = new InstanceConfig();
        instanceConfig.setFunctionDetails(functionDetails);
        // TODO: correctly implement function version and id
        instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
        instanceConfig.setFunctionId(UUID.randomUUID().toString());
        instanceConfig.setInstanceId(i + instanceIdOffset);
        instanceConfig.setMaxBufferedTuples(1024);
        instanceConfig.setPort(FunctionCommon.findAvailablePort());
        if (metricsPortStart != null) {
            int metricsPort = metricsPortStart + i;
            if (metricsPortStart < 0 || metricsPortStart > 65535) {
                throw new IllegalArgumentException("Metrics port need to be within the range of 0 and 65535");
            }
            instanceConfig.setMetricsPort(metricsPort);
        } else {
            instanceConfig.setMetricsPort(FunctionCommon.findAvailablePort());
        }
        instanceConfig.setClusterName("local");
        if (functionConfig != null) {
            instanceConfig.setMaxPendingAsyncRequests(functionConfig.getMaxPendingAsyncRequests());
            if (functionConfig.getExposePulsarAdminClientEnabled() != null) {
                instanceConfig.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
            }
        }
        RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, userCodeFile, null, runtimeFactory, instanceLivenessCheck);
        spawners.add(runtimeSpawner);
        runtimeSpawner.start();
    }
    Timer statusCheckTimer = new Timer();
    statusCheckTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            CompletableFuture<String>[] futures = new CompletableFuture[spawners.size()];
            int index = 0;
            for (RuntimeSpawner spawner : spawners) {
                futures[index] = spawner.getFunctionStatusAsJson(index);
                index++;
            }
            try {
                CompletableFuture.allOf(futures).get(5, TimeUnit.SECONDS);
                for (index = 0; index < futures.length; ++index) {
                    String json = futures[index].get();
                    Gson gson = new GsonBuilder().setPrettyPrinting().create();
                    log.info(gson.toJson(JsonParser.parseString(json)));
                }
            } catch (TimeoutException | InterruptedException | ExecutionException e) {
                log.error("Could not get status from all local instances");
            }
        }
    }, 30000, 30000);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread(() -> statusCheckTimer.cancel()));
}

Example 4 with ProcessRuntimeFactory

use of org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory in project incubator-pulsar by apache.

the class FunctionRuntimeManagerTest method testFunctionRuntimeFactoryConfigsBackwardsCompatibility.

@Test
public void testFunctionRuntimeFactoryConfigsBackwardsCompatibility() throws Exception {
    // Test kubernetes runtime
    WorkerConfig.KubernetesContainerFactory kubernetesContainerFactory = new WorkerConfig.KubernetesContainerFactory();
    kubernetesContainerFactory.setK8Uri("k8Uri");
    kubernetesContainerFactory.setJobNamespace("jobNamespace");
    kubernetesContainerFactory.setJobName("jobName");
    kubernetesContainerFactory.setPulsarDockerImageName("pulsarDockerImageName");
    kubernetesContainerFactory.setImagePullPolicy("imagePullPolicy");
    kubernetesContainerFactory.setPulsarRootDir("pulsarRootDir");
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setKubernetesContainerFactory(kubernetesContainerFactory);
    try (MockedConstruction<KubernetesRuntimeFactory> mocked = Mockito.mockConstruction(KubernetesRuntimeFactory.class, withSettings().defaultAnswer(CALLS_REAL_METHODS), (mockedKubernetesRuntimeFactory, context) -> {
        doNothing().when(mockedKubernetesRuntimeFactory).initialize(any(WorkerConfig.class), any(AuthenticationConfig.class), any(SecretsProviderConfigurator.class), any(), any(), any());
        doNothing().when(mockedKubernetesRuntimeFactory).setupClient();
        doReturn(true).when(mockedKubernetesRuntimeFactory).externallyManaged();
    })) {
        FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager(workerConfig, mock(PulsarWorkerService.class), mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        KubernetesRuntimeFactory kubernetesRuntimeFactory = (KubernetesRuntimeFactory) functionRuntimeManager.getRuntimeFactory();
        assertEquals(kubernetesRuntimeFactory.getK8Uri(), "k8Uri");
        assertEquals(kubernetesRuntimeFactory.getJobNamespace(), "jobNamespace");
        assertEquals(kubernetesRuntimeFactory.getPulsarDockerImageName(), "pulsarDockerImageName");
        assertEquals(kubernetesRuntimeFactory.getImagePullPolicy(), "imagePullPolicy");
        assertEquals(kubernetesRuntimeFactory.getPulsarRootDir(), "pulsarRootDir");
        // Test process runtime
        WorkerConfig.ProcessContainerFactory processContainerFactory = new WorkerConfig.ProcessContainerFactory();
        processContainerFactory.setExtraFunctionDependenciesDir("extraDependenciesDir");
        processContainerFactory.setLogDirectory("logDirectory");
        processContainerFactory.setPythonInstanceLocation("pythonInstanceLocation");
        processContainerFactory.setJavaInstanceJarLocation("javaInstanceJarLocation");
        workerConfig = new WorkerConfig();
        workerConfig.setProcessContainerFactory(processContainerFactory);
        functionRuntimeManager = new FunctionRuntimeManager(workerConfig, mock(PulsarWorkerService.class), mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        assertEquals(functionRuntimeManager.getRuntimeFactory().getClass(), ProcessRuntimeFactory.class);
        ProcessRuntimeFactory processRuntimeFactory = (ProcessRuntimeFactory) functionRuntimeManager.getRuntimeFactory();
        assertEquals(processRuntimeFactory.getExtraDependenciesDir(), "extraDependenciesDir");
        assertEquals(processRuntimeFactory.getLogDirectory(), "logDirectory/functions");
        assertEquals(processRuntimeFactory.getPythonInstanceFile(), "pythonInstanceLocation");
        assertEquals(processRuntimeFactory.getJavaInstanceJarFile(), "javaInstanceJarLocation");
        // Test thread runtime
        WorkerConfig.ThreadContainerFactory threadContainerFactory = new WorkerConfig.ThreadContainerFactory();
        threadContainerFactory.setThreadGroupName("threadGroupName");
        workerConfig = new WorkerConfig();
        workerConfig.setThreadContainerFactory(threadContainerFactory);
        workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
        functionRuntimeManager = new FunctionRuntimeManager(workerConfig, mock(PulsarWorkerService.class), mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        assertEquals(functionRuntimeManager.getRuntimeFactory().getClass(), ThreadRuntimeFactory.class);
        ThreadRuntimeFactory threadRuntimeFactory = (ThreadRuntimeFactory) functionRuntimeManager.getRuntimeFactory();
        assertEquals(threadRuntimeFactory.getThreadGroup().getName(), "threadGroupName");
    }
}

Example 5 with ProcessRuntimeFactory

use of org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory in project pulsar by apache.

the class LocalRunner method startProcessMode.

private void startProcessMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails, int parallelism, int instanceIdOffset, String serviceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, String userCodeFile) throws Exception {
    SecretsProviderConfigurator secretsProviderConfigurator = getSecretsProviderConfigurator();
    runtimeFactory = new ProcessRuntimeFactory(serviceUrl, webServiceUrl, stateStorageServiceUrl, authConfig, null, /* java instance jar file */
    null, /* python instance file */
    null, /* log directory */
    null, /* extra dependencies dir */
    narExtractionDirectory, /* nar extraction dir */
    secretsProviderConfigurator, false, Optional.empty(), Optional.empty());
    for (int i = 0; i < parallelism; ++i) {
        InstanceConfig instanceConfig = new InstanceConfig();
        instanceConfig.setFunctionDetails(functionDetails);
        // TODO: correctly implement function version and id
        instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
        instanceConfig.setFunctionId(UUID.randomUUID().toString());
        instanceConfig.setInstanceId(i + instanceIdOffset);
        instanceConfig.setMaxBufferedTuples(1024);
        instanceConfig.setPort(FunctionCommon.findAvailablePort());
        if (metricsPortStart != null) {
            int metricsPort = metricsPortStart + i;
            if (metricsPortStart < 0 || metricsPortStart > 65535) {
                throw new IllegalArgumentException("Metrics port need to be within the range of 0 and 65535");
            }
            instanceConfig.setMetricsPort(metricsPort);
        } else {
            instanceConfig.setMetricsPort(FunctionCommon.findAvailablePort());
        }
        instanceConfig.setClusterName("local");
        if (functionConfig != null) {
            instanceConfig.setMaxPendingAsyncRequests(functionConfig.getMaxPendingAsyncRequests());
            if (functionConfig.getExposePulsarAdminClientEnabled() != null) {
                instanceConfig.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
            }
        }
        RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, userCodeFile, null, runtimeFactory, instanceLivenessCheck);
        spawners.add(runtimeSpawner);
        runtimeSpawner.start();
    }
    Timer statusCheckTimer = new Timer();
    statusCheckTimer.scheduleAtFixedRate(new TimerTask() {

        @Override
        public void run() {
            CompletableFuture<String>[] futures = new CompletableFuture[spawners.size()];
            int index = 0;
            for (RuntimeSpawner spawner : spawners) {
                futures[index] = spawner.getFunctionStatusAsJson(index);
                index++;
            }
            try {
                CompletableFuture.allOf(futures).get(5, TimeUnit.SECONDS);
                for (index = 0; index < futures.length; ++index) {
                    String json = futures[index].get();
                    Gson gson = new GsonBuilder().setPrettyPrinting().create();
                    log.info(gson.toJson(JsonParser.parseString(json)));
                }
            } catch (TimeoutException | InterruptedException | ExecutionException e) {
                log.error("Could not get status from all local instances");
            }
        }
    }, 30000, 30000);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread(() -> statusCheckTimer.cancel()));
}