Examples with ThreadRuntimeFactoryConfig org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig used on opensource projects

Example 1 with ThreadRuntimeFactoryConfig

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig in project pulsar by apache.

the class PulsarFunctionE2ESecurityTest method createPulsarFunctionWorker.

private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration config) {
    System.setProperty(JAVA_INSTANCE_JAR_PROPERTY, FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath());
    workerConfig = new WorkerConfig();
    tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
    tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
    workerConfig.setPulsarFunctionsNamespace(pulsarFunctionsNamespace);
    workerConfig.setSchedulerClassName(org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler.class.getName());
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("use"), Map.class));
    // worker talks to local broker
    workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort().get());
    workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + config.getWebServicePort().get());
    workerConfig.setFailureCheckFreqMs(100);
    workerConfig.setNumFunctionPackageReplicas(1);
    workerConfig.setClusterCoordinationTopicName("coordinate");
    workerConfig.setFunctionAssignmentTopicName("assignment");
    workerConfig.setFunctionMetadataTopicName("metadata");
    workerConfig.setInstanceLivenessCheckFreqMs(100);
    workerConfig.setWorkerPort(0);
    workerConfig.setPulsarFunctionsCluster(config.getClusterName());
    String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
    this.workerId = "c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort();
    workerConfig.setWorkerHostname(hostname);
    workerConfig.setWorkerId(workerId);
    workerConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName());
    workerConfig.setBrokerClientAuthenticationParameters(String.format("token:%s", adminToken));
    workerConfig.setAuthenticationEnabled(config.isAuthenticationEnabled());
    workerConfig.setAuthenticationProviders(config.getAuthenticationProviders());
    workerConfig.setAuthorizationEnabled(config.isAuthorizationEnabled());
    workerConfig.setAuthorizationProvider(config.getAuthorizationProvider());
    PulsarWorkerService workerService = new PulsarWorkerService();
    workerService.init(workerConfig, null, false);
    return workerService;
}

Example 2 with ThreadRuntimeFactoryConfig

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig in project pulsar by apache.

the class PulsarWorkerAssignmentTest method createPulsarFunctionWorker.

private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration config) {
    workerConfig = new WorkerConfig();
    tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
    tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
    workerConfig.setPulsarFunctionsNamespace(pulsarFunctionsNamespace);
    workerConfig.setSchedulerClassName(org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler.class.getName());
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("use"), Map.class));
    // worker talks to local broker
    workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort().get());
    workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + config.getWebServicePort().get());
    workerConfig.setFailureCheckFreqMs(100);
    workerConfig.setNumFunctionPackageReplicas(1);
    workerConfig.setClusterCoordinationTopicName("coordinate");
    workerConfig.setFunctionAssignmentTopicName("assignment");
    workerConfig.setFunctionMetadataTopicName("metadata");
    workerConfig.setInstanceLivenessCheckFreqMs(100);
    workerConfig.setWorkerPort(0);
    workerConfig.setPulsarFunctionsCluster(config.getClusterName());
    final String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
    workerId = "c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort();
    workerConfig.setWorkerHostname(hostname);
    workerConfig.setWorkerId(workerId);
    workerConfig.setTopicCompactionFrequencySec(1);
    PulsarWorkerService workerService = new PulsarWorkerService();
    workerService.init(workerConfig, null, false);
    return workerService;
}

Example 3 with ThreadRuntimeFactoryConfig

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig in project pulsar by apache.

the class AbstractPulsarE2ETest method createPulsarFunctionWorker.

private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration config) throws IOException {
    System.setProperty(JAVA_INSTANCE_JAR_PROPERTY, FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath());
    workerConfig = new WorkerConfig();
    tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName());
    tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig);
    workerConfig.setPulsarFunctionsNamespace(pulsarFunctionsNamespace);
    workerConfig.setSchedulerClassName(org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler.class.getName());
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(// worker talks to local broker
    ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("use"), Map.class));
    workerConfig.setFailureCheckFreqMs(100);
    workerConfig.setNumFunctionPackageReplicas(1);
    workerConfig.setClusterCoordinationTopicName("coordinate");
    workerConfig.setFunctionAssignmentTopicName("assignment");
    workerConfig.setFunctionMetadataTopicName("metadata");
    workerConfig.setInstanceLivenessCheckFreqMs(100);
    workerConfig.setWorkerPort(0);
    workerConfig.setPulsarFunctionsCluster(config.getClusterName());
    String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
    this.workerId = "c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort();
    workerConfig.setWorkerHostname(hostname);
    workerConfig.setWorkerId(workerId);
    workerConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
    workerConfig.setBrokerClientAuthenticationParameters(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH));
    workerConfig.setUseTls(true);
    workerConfig.setTlsAllowInsecureConnection(true);
    workerConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
    workerConfig.setAuthenticationEnabled(true);
    workerConfig.setAuthorizationEnabled(true);
    PulsarWorkerService workerService = new PulsarWorkerService();
    workerService.init(workerConfig, null, false);
    return workerService;
}

Example 4 with ThreadRuntimeFactoryConfig

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig in project pulsar by apache.

the class PulsarFunctionAdminTest method createPulsarFunctionWorker.

private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration config) {
    workerConfig = new WorkerConfig();
    workerConfig.setPulsarFunctionsNamespace(pulsarFunctionsNamespace);
    workerConfig.setSchedulerClassName(org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler.class.getName());
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("use"), Map.class));
    // worker talks to local broker
    workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePortTls().get());
    workerConfig.setPulsarWebServiceUrl("https://127.0.0.1:" + config.getWebServicePortTls().get());
    workerConfig.setFailureCheckFreqMs(100);
    workerConfig.setNumFunctionPackageReplicas(1);
    workerConfig.setClusterCoordinationTopicName("coordinate");
    workerConfig.setFunctionAssignmentTopicName("assignment");
    workerConfig.setFunctionMetadataTopicName("metadata");
    workerConfig.setInstanceLivenessCheckFreqMs(100);
    workerConfig.setWorkerPort(0);
    workerConfig.setPulsarFunctionsCluster(config.getClusterName());
    String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
    workerConfig.setWorkerHostname(hostname);
    workerConfig.setWorkerId("c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort());
    workerConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName());
    workerConfig.setBrokerClientAuthenticationParameters(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH));
    workerConfig.setUseTls(true);
    workerConfig.setTlsAllowInsecureConnection(true);
    workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
    PulsarWorkerService workerService = new PulsarWorkerService();
    workerService.init(workerConfig, null, false);
    return workerService;
}

Example 5 with ThreadRuntimeFactoryConfig

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig in project pulsar by apache.

the class FunctionRuntimeManagerTest method testProcessAssignmentUpdateModifyFunctions.

@Test
public void testProcessAssignmentUpdateModifyFunctions() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    PulsarClient pulsarClient = mock(PulsarClient.class);
    ReaderBuilder readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(anyBoolean());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin();
    try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
        mockRuntimeFactory(runtimeFactoryMockedStatic);
        // test new assignment update functions
        FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner());
        doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class));
        functionRuntimeManager.setFunctionActioner(functionActioner);
        Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
        Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
        Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
        Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-2").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
        // add existing assignments
        functionRuntimeManager.setAssignment(assignment1);
        functionRuntimeManager.setAssignment(assignment2);
        reset(functionActioner);
        Function.Assignment assignment3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
        functionRuntimeManager.functionRuntimeInfos.put("test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()));
        functionRuntimeManager.functionRuntimeInfos.put("test-tenant/test-namespace/func-2:0", new FunctionRuntimeInfo().setFunctionInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()));
        functionRuntimeManager.processAssignment(assignment1);
        functionRuntimeManager.processAssignment(assignment3);
        verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class));
        // make sure terminate is not called since this is a update operation
        verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).stopFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2)));
        verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).startFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2)));
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2);
        assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment3);
        reset(functionActioner);
        // add a stop
        Function.FunctionMetaData.Builder function2StoppedBldr = function2.toBuilder();
        function2StoppedBldr.putInstanceStates(0, Function.FunctionState.STOPPED);
        Function.FunctionMetaData function2Stopped = function2StoppedBldr.build();
        Function.Assignment assignment4 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Stopped).setInstanceId(0).build()).build();
        functionRuntimeManager.processAssignment(assignment4);
        verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class));
        // make sure terminate is not called since this is a update operation
        verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).stopFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2)));
        verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class));
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2);
        assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment4);
    }
}