Search in sources :

Example 1 with RuntimeFactory

use of org.apache.pulsar.functions.runtime.RuntimeFactory in project pulsar by apache.

the class FunctionRuntimeManagerTest method testKubernetesFunctionInstancesRestart.

@Test
public void testKubernetesFunctionInstancesRestart() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setFunctionAssignmentTopicName("assignments");
    WorkerConfig.KubernetesContainerFactory kubernetesContainerFactory = new WorkerConfig.KubernetesContainerFactory();
    workerConfig.setKubernetesContainerFactory(kubernetesContainerFactory);
    try (MockedConstruction<KubernetesRuntimeFactory> mocked = Mockito.mockConstruction(KubernetesRuntimeFactory.class, (mockedKubernetesRuntimeFactory, context) -> {
        doNothing().when(mockedKubernetesRuntimeFactory).initialize(any(WorkerConfig.class), any(AuthenticationConfig.class), any(SecretsProviderConfigurator.class), any(), any(), any());
        doNothing().when(mockedKubernetesRuntimeFactory).setupClient();
        doReturn(true).when(mockedKubernetesRuntimeFactory).externallyManaged();
    })) {
        PulsarWorkerService workerService = mock(PulsarWorkerService.class);
        // mock pulsarAdmin sources sinks functions
        PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class);
        Sources sources = mock(Sources.class);
        doNothing().when(sources).restartSource(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
        doReturn(sources).when(pulsarAdmin).sources();
        Sinks sinks = mock(Sinks.class);
        doReturn(sinks).when(pulsarAdmin).sinks();
        Functions functions = mock(Functions.class);
        doNothing().when(functions).restartFunction(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
        doReturn(functions).when(pulsarAdmin).functions();
        doReturn(pulsarAdmin).when(workerService).getFunctionAdmin();
        try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
            mockRuntimeFactory(runtimeFactoryMockedStatic);
            List<WorkerInfo> workerInfos = new LinkedList<>();
            workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0));
            workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0));
            MembershipManager membershipManager = mock(MembershipManager.class);
            doReturn(workerInfos).when(membershipManager).getCurrentMembership();
            // build three types of FunctionMetaData
            Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("function").setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build();
            Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("source").setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build();
            Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("sink").setComponentType(Function.FunctionDetails.ComponentType.SINK)).build();
            FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), membershipManager, mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
            // verify restart function/source/sink using different assignment
            verifyRestart(functionRuntimeManager, function, "worker-1", true, false);
            verifyRestart(functionRuntimeManager, function, "worker-2", true, true);
            verifyRestart(functionRuntimeManager, source, "worker-1", true, false);
            verifyRestart(functionRuntimeManager, source, "worker-2", true, true);
            verifyRestart(functionRuntimeManager, sink, "worker-1", true, false);
            verifyRestart(functionRuntimeManager, sink, "worker-2", true, true);
        }
    }
}
Also used : Sinks(org.apache.pulsar.client.admin.Sinks) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) WorkerInfo(org.apache.pulsar.common.functions.WorkerInfo) Functions(org.apache.pulsar.client.admin.Functions) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) Sources(org.apache.pulsar.client.admin.Sources) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) ProcessRuntimeFactory(org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) LinkedList(java.util.LinkedList) AuthenticationConfig(org.apache.pulsar.functions.instance.AuthenticationConfig) Function(org.apache.pulsar.functions.proto.Function) SecretsProviderConfigurator(org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator) Test(org.testng.annotations.Test)

Example 2 with RuntimeFactory

use of org.apache.pulsar.functions.runtime.RuntimeFactory in project pulsar by apache.

the class FunctionRuntimeManagerTest method testProcessAssignmentUpdateModifyFunctions.

@Test
public void testProcessAssignmentUpdateModifyFunctions() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    PulsarClient pulsarClient = mock(PulsarClient.class);
    ReaderBuilder readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(anyBoolean());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin();
    try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
        mockRuntimeFactory(runtimeFactoryMockedStatic);
        // test new assignment update functions
        FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner());
        doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class));
        functionRuntimeManager.setFunctionActioner(functionActioner);
        Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
        Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
        Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
        Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-2").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
        // add existing assignments
        functionRuntimeManager.setAssignment(assignment1);
        functionRuntimeManager.setAssignment(assignment2);
        reset(functionActioner);
        Function.Assignment assignment3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
        functionRuntimeManager.functionRuntimeInfos.put("test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()));
        functionRuntimeManager.functionRuntimeInfos.put("test-tenant/test-namespace/func-2:0", new FunctionRuntimeInfo().setFunctionInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()));
        functionRuntimeManager.processAssignment(assignment1);
        functionRuntimeManager.processAssignment(assignment3);
        verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class));
        // make sure terminate is not called since this is a update operation
        verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).stopFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2)));
        verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).startFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2)));
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2);
        assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment3);
        reset(functionActioner);
        // add a stop
        Function.FunctionMetaData.Builder function2StoppedBldr = function2.toBuilder();
        function2StoppedBldr.putInstanceStates(0, Function.FunctionState.STOPPED);
        Function.FunctionMetaData function2Stopped = function2StoppedBldr.build();
        Function.Assignment assignment4 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Stopped).setInstanceId(0).build()).build();
        functionRuntimeManager.processAssignment(assignment4);
        verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class));
        // make sure terminate is not called since this is a update operation
        verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).stopFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function2)));
        verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class));
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 2);
        assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-2:0"), assignment4);
    }
}
Also used : ArgumentMatchers(org.mockito.ArgumentMatchers) KubernetesRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactoryConfig) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Test(org.testng.annotations.Test) Mockito.argThat(org.mockito.Mockito.argThat) Unpooled(io.netty.buffer.Unpooled) CALLS_REAL_METHODS(org.mockito.Mockito.CALLS_REAL_METHODS) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Mockito.anyBoolean(org.mockito.Mockito.anyBoolean) Map(java.util.Map) KubernetesRuntime(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntime) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) WorkerInfo(org.apache.pulsar.common.functions.WorkerInfo) Mockito.doReturn(org.mockito.Mockito.doReturn) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) MockedConstruction(org.mockito.MockedConstruction) Function(org.apache.pulsar.functions.proto.Function) Mockito.doNothing(org.mockito.Mockito.doNothing) Assert.assertNotNull(org.testng.Assert.assertNotNull) Sinks(org.apache.pulsar.client.admin.Sinks) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) MockedStatic(org.mockito.MockedStatic) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) Mockito.withSettings(org.mockito.Mockito.withSettings) Mockito.any(org.mockito.Mockito.any) Mockito.mock(org.mockito.Mockito.mock) Assert.assertNull(org.testng.Assert.assertNull) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AuthenticationConfig(org.apache.pulsar.functions.instance.AuthenticationConfig) ProcessRuntimeFactory(org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ImmutableList(com.google.common.collect.ImmutableList) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) LinkedList(java.util.LinkedList) Functions(org.apache.pulsar.client.admin.Functions) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Mockito.anyString(org.mockito.Mockito.anyString) Iterator(java.util.Iterator) Sources(org.apache.pulsar.client.admin.Sources) Assert.fail(org.testng.Assert.fail) Reader(org.apache.pulsar.client.api.Reader) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) Namespace(org.apache.distributedlog.api.namespace.Namespace) Mockito(org.mockito.Mockito) MessageId(org.apache.pulsar.client.api.MessageId) SecretsProviderConfigurator(org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator) FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon) Assert.assertTrue(org.testng.Assert.assertTrue) ReaderBuilder(org.apache.pulsar.client.api.ReaderBuilder) Mockito.reset(org.mockito.Mockito.reset) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) Reader(org.apache.pulsar.client.api.Reader) ReaderBuilder(org.apache.pulsar.client.api.ReaderBuilder) Function(org.apache.pulsar.functions.proto.Function) PulsarClient(org.apache.pulsar.client.api.PulsarClient) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) ProcessRuntimeFactory(org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Namespace(org.apache.distributedlog.api.namespace.Namespace) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.testng.annotations.Test)

Example 3 with RuntimeFactory

use of org.apache.pulsar.functions.runtime.RuntimeFactory in project pulsar by apache.

the class FunctionRuntimeManagerTest method testProcessAssignmentUpdateDeleteFunctions.

@Test
public void testProcessAssignmentUpdateDeleteFunctions() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    PulsarClient pulsarClient = mock(PulsarClient.class);
    ReaderBuilder readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(anyBoolean());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin();
    try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
        mockRuntimeFactory(runtimeFactoryMockedStatic);
        // test new assignment delete functions
        FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
        FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner());
        doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class));
        functionRuntimeManager.setFunctionActioner(functionActioner);
        Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
        Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
        // Delete this assignment
        Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
        Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-2").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
        // add existing assignments
        functionRuntimeManager.setAssignment(assignment1);
        functionRuntimeManager.setAssignment(assignment2);
        reset(functionRuntimeManager);
        functionRuntimeManager.functionRuntimeInfos.put("test-tenant/test-namespace/func-1:0", new FunctionRuntimeInfo().setFunctionInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()));
        functionRuntimeManager.processAssignment(assignment1);
        functionRuntimeManager.processAssignment(assignment2);
        functionRuntimeManager.deleteAssignment(FunctionCommon.getFullyQualifiedInstanceId(assignment1.getInstance()));
        verify(functionRuntimeManager, times(0)).setAssignment(any(Function.Assignment.class));
        verify(functionRuntimeManager, times(1)).deleteAssignment(any(String.class));
        assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1);
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-2").get("test-tenant/test-namespace/func-2:0"), assignment2);
        verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner, times(1)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner).terminateFunction(argThat(functionRuntimeInfo -> functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().equals(function1)));
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 0);
    }
}
Also used : ArgumentMatchers(org.mockito.ArgumentMatchers) KubernetesRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactoryConfig) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) Test(org.testng.annotations.Test) Mockito.argThat(org.mockito.Mockito.argThat) Unpooled(io.netty.buffer.Unpooled) CALLS_REAL_METHODS(org.mockito.Mockito.CALLS_REAL_METHODS) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Mockito.anyBoolean(org.mockito.Mockito.anyBoolean) Map(java.util.Map) KubernetesRuntime(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntime) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) WorkerInfo(org.apache.pulsar.common.functions.WorkerInfo) Mockito.doReturn(org.mockito.Mockito.doReturn) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) MockedConstruction(org.mockito.MockedConstruction) Function(org.apache.pulsar.functions.proto.Function) Mockito.doNothing(org.mockito.Mockito.doNothing) Assert.assertNotNull(org.testng.Assert.assertNotNull) Sinks(org.apache.pulsar.client.admin.Sinks) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) MockedStatic(org.mockito.MockedStatic) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) Mockito.withSettings(org.mockito.Mockito.withSettings) Mockito.any(org.mockito.Mockito.any) Mockito.mock(org.mockito.Mockito.mock) Assert.assertNull(org.testng.Assert.assertNull) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AuthenticationConfig(org.apache.pulsar.functions.instance.AuthenticationConfig) ProcessRuntimeFactory(org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ImmutableList(com.google.common.collect.ImmutableList) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) LinkedList(java.util.LinkedList) Functions(org.apache.pulsar.client.admin.Functions) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Mockito.anyString(org.mockito.Mockito.anyString) Iterator(java.util.Iterator) Sources(org.apache.pulsar.client.admin.Sources) Assert.fail(org.testng.Assert.fail) Reader(org.apache.pulsar.client.api.Reader) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) Namespace(org.apache.distributedlog.api.namespace.Namespace) Mockito(org.mockito.Mockito) MessageId(org.apache.pulsar.client.api.MessageId) SecretsProviderConfigurator(org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator) FunctionCommon(org.apache.pulsar.functions.utils.FunctionCommon) Assert.assertTrue(org.testng.Assert.assertTrue) ReaderBuilder(org.apache.pulsar.client.api.ReaderBuilder) Mockito.reset(org.mockito.Mockito.reset) ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Reader(org.apache.pulsar.client.api.Reader) Mockito.anyString(org.mockito.Mockito.anyString) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) ProcessRuntimeFactory(org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) ReaderBuilder(org.apache.pulsar.client.api.ReaderBuilder) Function(org.apache.pulsar.functions.proto.Function) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.testng.annotations.Test)

Example 4 with RuntimeFactory

use of org.apache.pulsar.functions.runtime.RuntimeFactory in project pulsar by apache.

the class FunctionRuntimeManagerTest method testReassignment.

@Test
public void testReassignment() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    PulsarClient pulsarClient = mock(PulsarClient.class);
    ReaderBuilder readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(anyBoolean());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin();
    try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
        mockRuntimeFactory(runtimeFactoryMockedStatic);
        // test new assignment update functions
        FunctionRuntimeManager functionRuntimeManager = new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class));
        FunctionActioner functionActioner = spy(functionRuntimeManager.getFunctionActioner());
        doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class));
        functionRuntimeManager.setFunctionActioner(functionActioner);
        Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
        Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
        /**
         * Test transfer from me to other worker *
         */
        // add existing assignments
        functionRuntimeManager.setAssignment(assignment1);
        // new assignment with different worker
        Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-2").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
        FunctionRuntimeInfo functionRuntimeInfo = new FunctionRuntimeInfo().setFunctionInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build());
        functionRuntimeManager.functionRuntimeInfos.put("test-tenant/test-namespace/func-1:0", functionRuntimeInfo);
        functionRuntimeManager.processAssignment(assignment2);
        verify(functionActioner, times(0)).startFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner, times(1)).stopFunction(any(FunctionRuntimeInfo.class));
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-2").get("test-tenant/test-namespace/func-1:0"), assignment2);
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 0);
        assertNull(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"));
        /**
         * Test transfer from other worker to me *
         */
        reset(functionActioner);
        doNothing().when(functionActioner).startFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).stopFunction(any(FunctionRuntimeInfo.class));
        doNothing().when(functionActioner).terminateFunction(any(FunctionRuntimeInfo.class));
        functionRuntimeManager.setFunctionActioner(functionActioner);
        Function.Assignment assignment3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
        functionRuntimeManager.processAssignment(assignment3);
        verify(functionActioner, times(1)).startFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner, times(0)).terminateFunction(any(FunctionRuntimeInfo.class));
        verify(functionActioner, times(0)).stopFunction(any(FunctionRuntimeInfo.class));
        assertEquals(functionRuntimeManager.workerIdToAssignments.get("worker-1").get("test-tenant/test-namespace/func-1:0"), assignment3);
        assertNull(functionRuntimeManager.workerIdToAssignments.get("worker-2"));
        assertEquals(functionRuntimeManager.functionRuntimeInfos.size(), 1);
        assertEquals(functionRuntimeManager.functionRuntimeInfos.get("test-tenant/test-namespace/func-1:0"), functionRuntimeInfo);
    }
}
Also used : ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Reader(org.apache.pulsar.client.api.Reader) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) KubernetesRuntimeFactory(org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory) ProcessRuntimeFactory(org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Namespace(org.apache.distributedlog.api.namespace.Namespace) ReaderBuilder(org.apache.pulsar.client.api.ReaderBuilder) Function(org.apache.pulsar.functions.proto.Function) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.testng.annotations.Test)

Example 5 with RuntimeFactory

use of org.apache.pulsar.functions.runtime.RuntimeFactory in project pulsar by apache.

the class FunctionActionerTest method testStartFunctionWithPkgUrl.

@Test
public void testStartFunctionWithPkgUrl() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl("pulsar://localhost:6650");
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setFunctionAssignmentTopicName("assignments");
    String downloadDir = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
    workerConfig.setDownloadDirectory(downloadDir);
    RuntimeFactory factory = mock(RuntimeFactory.class);
    Runtime runtime = mock(Runtime.class);
    doReturn(runtime).when(factory).createContainer(any(), any(), any(), any());
    doNothing().when(runtime).start();
    Namespace dlogNamespace = mock(Namespace.class);
    final String exceptionMsg = "dl namespace not-found";
    doThrow(new IllegalArgumentException(exceptionMsg)).when(dlogNamespace).openLog(any());
    @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class));
    // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call
    // RuntimeSpawner
    String pkgPathLocation = FILE + ":/user/my-file.jar";
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath(pkgPathLocation).build()).build();
    Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build();
    FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class);
    doReturn(instance).when(functionRuntimeInfo).getFunctionInstance();
    actioner.startFunction(functionRuntimeInfo);
    verify(runtime, times(1)).start();
    // (2) test with http-url, downloading file from http should fail with UnknownHostException due to invalid url
    pkgPathLocation = "http://invalid/my-file.jar";
    function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath(pkgPathLocation).build()).build();
    instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build();
    functionRuntimeInfo = mock(FunctionRuntimeInfo.class);
    doReturn(instance).when(functionRuntimeInfo).getFunctionInstance();
    doThrow(new IllegalStateException("StartupException")).when(functionRuntimeInfo).setStartupException(any());
    try {
        actioner.startFunction(functionRuntimeInfo);
        fail();
    } catch (IllegalStateException ex) {
        assertEquals(ex.getMessage(), "StartupException");
    }
}
Also used : ThreadRuntimeFactoryConfig(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) RuntimeFactory(org.apache.pulsar.functions.runtime.RuntimeFactory) Namespace(org.apache.distributedlog.api.namespace.Namespace) Function(org.apache.pulsar.functions.proto.Function) Runtime(org.apache.pulsar.functions.runtime.Runtime) ThreadRuntimeFactory(org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory) Map(java.util.Map) Test(org.testng.annotations.Test)

Aggregations

PulsarAdmin (org.apache.pulsar.client.admin.PulsarAdmin)32 Function (org.apache.pulsar.functions.proto.Function)32 RuntimeFactory (org.apache.pulsar.functions.runtime.RuntimeFactory)32 ThreadRuntimeFactory (org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory)32 Test (org.testng.annotations.Test)32 Map (java.util.Map)29 Namespace (org.apache.distributedlog.api.namespace.Namespace)26 ThreadRuntimeFactoryConfig (org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig)26 KubernetesRuntimeFactory (org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory)24 ProcessRuntimeFactory (org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory)24 HashMap (java.util.HashMap)21 LinkedList (java.util.LinkedList)18 Functions (org.apache.pulsar.client.admin.Functions)18 Sinks (org.apache.pulsar.client.admin.Sinks)18 Sources (org.apache.pulsar.client.admin.Sources)18 PulsarClient (org.apache.pulsar.client.api.PulsarClient)18 Reader (org.apache.pulsar.client.api.Reader)18 ReaderBuilder (org.apache.pulsar.client.api.ReaderBuilder)18 WorkerInfo (org.apache.pulsar.common.functions.WorkerInfo)18 AuthenticationConfig (org.apache.pulsar.functions.instance.AuthenticationConfig)18