Examples with WorkerInfo org.apache.pulsar.common.functions.WorkerInfo used on opensource projects

Example 16 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by apache.

the class WorkerImpl method buildRedirectUriForDrainRelatedOp.

private URI buildRedirectUriForDrainRelatedOp(final URI uri, String workerId) {
    // The incoming URI could be a leader URI (sent to a non-leader), or a non-leader URI.
    // Leader URI example: “/admin/v2/worker/leader/drain?workerId=<WORKER_ID>”
    // Non-leader URI example: “/admin/v2/worker/drain”
    // Use the leader-URI path in both cases for the redirect to the leader.
    String leaderPath = "admin/v2/worker/leader/drain";
    WorkerInfo workerInfo = worker().getMembershipManager().getLeader();
    URI redirect = UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).replacePath(leaderPath).replaceQueryParam("workerId", workerId).build();
    return redirect;
}

Example 17 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class FunctionRuntimeManagerTest method testThreadFunctionInstancesRestart.

@Test
public void testThreadFunctionInstancesRestart() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setFunctionAssignmentTopicName("assignments");
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    // mock pulsarAdmin sources sinks functions
    PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class);
    Sources sources = mock(Sources.class);
    doNothing().when(sources).restartSource(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    doReturn(sources).when(pulsarAdmin).sources();
    Sinks sinks = mock(Sinks.class);
    doReturn(sinks).when(pulsarAdmin).sinks();
    Functions functions = mock(Functions.class);
    doNothing().when(functions).restartFunction(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    doReturn(functions).when(pulsarAdmin).functions();
    doReturn(pulsarAdmin).when(workerService).getFunctionAdmin();
    try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
        mockRuntimeFactory(runtimeFactoryMockedStatic);
        List<WorkerInfo> workerInfos = new LinkedList<>();
        workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0));
        workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0));
        MembershipManager membershipManager = mock(MembershipManager.class);
        doReturn(workerInfos).when(membershipManager).getCurrentMembership();
        // build three types of FunctionMetaData
        Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("function").setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build();
        Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("source").setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build();
        Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("sink").setComponentType(Function.FunctionDetails.ComponentType.SINK)).build();
        FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), membershipManager, mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
        // verify restart function/source/sink using different assignment
        verifyRestart(functionRuntimeManager, function, "worker-1", false, false);
        verifyRestart(functionRuntimeManager, function, "worker-2", false, true);
        verifyRestart(functionRuntimeManager, source, "worker-1", false, false);
        verifyRestart(functionRuntimeManager, source, "worker-2", false, true);
        verifyRestart(functionRuntimeManager, sink, "worker-1", false, false);
        verifyRestart(functionRuntimeManager, sink, "worker-2", false, true);
    }
}

Example 18 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class MembershipManagerTest method testCheckFailuresSomeUnassigned.

@Test
public void testCheckFailuresSomeUnassigned() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl("pulsar://localhost:6650");
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setRescheduleTimeoutMs(30000);
    SchedulerManager schedulerManager = mock(SchedulerManager.class);
    PulsarClient pulsarClient = mockPulsarClient();
    ReaderBuilder<byte[]> readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(true);
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(workerConfig).when(workerService).getWorkerConfig();
    PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class);
    doReturn(pulsarAdmin).when(workerService).getFunctionAdmin();
    FunctionMetaDataManager functionMetaDataManager = mock(FunctionMetaDataManager.class);
    FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), functionMetaDataManager, mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
    MembershipManager membershipManager = spy(new MembershipManager(workerService, mockPulsarClient(), pulsarAdmin));
    List<WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(WorkerInfo.of("worker-1", "host-1", 8000));
    workerInfoList.add(WorkerInfo.of("worker-2", "host-2", 8001));
    Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setParallelism(1).setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setParallelism(1).setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
    List<Function.FunctionMetaData> metaDataList = new LinkedList<>();
    metaDataList.add(function1);
    metaDataList.add(function2);
    Mockito.doReturn(metaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    // add existing assignments
    functionRuntimeManager.setAssignment(assignment1);
    membershipManager.checkFailures(functionMetaDataManager, functionRuntimeManager, schedulerManager);
    verify(schedulerManager, times(0)).schedule();
    verify(functionRuntimeManager, times(0)).removeAssignments(any());
    assertEquals(membershipManager.unsignedFunctionDurations.size(), 1);
    Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build();
    assertNotNull(membershipManager.unsignedFunctionDurations.get(instance));
    membershipManager.unsignedFunctionDurations.put(instance, membershipManager.unsignedFunctionDurations.get(instance) - 30001);
    membershipManager.checkFailures(functionMetaDataManager, functionRuntimeManager, schedulerManager);
    verify(schedulerManager, times(1)).schedule();
    verify(functionRuntimeManager, times(0)).removeAssignments(any());
}

Example 19 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class MembershipManagerTest method testHeartBeatFunctionWorkerDown.

@Test
public void testHeartBeatFunctionWorkerDown() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl("pulsar://localhost:6650");
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setRescheduleTimeoutMs(30000);
    SchedulerManager schedulerManager = mock(SchedulerManager.class);
    PulsarClient pulsarClient = mockPulsarClient();
    ReaderBuilder<byte[]> readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(true);
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(workerConfig).when(workerService).getWorkerConfig();
    PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class);
    doReturn(mock(PulsarAdmin.class)).when(workerService).getFunctionAdmin();
    FunctionMetaDataManager functionMetaDataManager = mock(FunctionMetaDataManager.class);
    FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), functionMetaDataManager, mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
    MembershipManager membershipManager = spy(new MembershipManager(workerService, mockPulsarClient(), pulsarAdmin));
    List<WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(WorkerInfo.of("worker-1", "host-1", 8000));
    // make worker-2 unavailable
    // workerInfoList.add(WorkerInfo.of("worker-2", "host-2", 8001));
    Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setParallelism(1).setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setParallelism(1).setTenant(SchedulerManager.HEARTBEAT_TENANT).setNamespace(SchedulerManager.HEARTBEAT_NAMESPACE).setName("worker-2")).build();
    List<Function.FunctionMetaData> metaDataList = new LinkedList<>();
    metaDataList.add(function1);
    metaDataList.add(function2);
    Mockito.doReturn(metaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-2").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    // add existing assignments
    functionRuntimeManager.setAssignment(assignment1);
    functionRuntimeManager.setAssignment(assignment2);
    membershipManager.checkFailures(functionMetaDataManager, functionRuntimeManager, schedulerManager);
    verify(schedulerManager, times(0)).schedule();
    verify(functionRuntimeManager, times(0)).removeAssignments(any());
    assertEquals(membershipManager.unsignedFunctionDurations.size(), 0);
}

Example 20 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class MembershipManagerTest method testCheckFailuresNoFailures.

@Test
public void testCheckFailuresNoFailures() throws Exception {
    SchedulerManager schedulerManager = mock(SchedulerManager.class);
    PulsarClient pulsarClient = mockPulsarClient();
    ReaderBuilder<byte[]> readerBuilder = mock(ReaderBuilder.class);
    doReturn(readerBuilder).when(pulsarClient).newReader();
    doReturn(readerBuilder).when(readerBuilder).topic(anyString());
    doReturn(readerBuilder).when(readerBuilder).readCompacted(true);
    doReturn(readerBuilder).when(readerBuilder).startMessageId(any());
    doReturn(mock(Reader.class)).when(readerBuilder).create();
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    doReturn(pulsarClient).when(workerService).getClient();
    doReturn(workerConfig).when(workerService).getWorkerConfig();
    PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class);
    doReturn(pulsarAdmin).when(workerService).getFunctionAdmin();
    FunctionMetaDataManager functionMetaDataManager = mock(FunctionMetaDataManager.class);
    FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), mock(MembershipManager.class), mock(ConnectorsManager.class), mock(FunctionsManager.class), functionMetaDataManager, mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
    MembershipManager membershipManager = spy(new MembershipManager(workerService, pulsarClient, pulsarAdmin));
    List<WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(WorkerInfo.of("worker-1", "host-1", 8000));
    workerInfoList.add(WorkerInfo.of("worker-2", "host-2", 8001));
    Mockito.doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-2")).build();
    List<Function.FunctionMetaData> metaDataList = new LinkedList<>();
    metaDataList.add(function1);
    metaDataList.add(function2);
    Mockito.doReturn(metaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-2").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    // add existing assignments
    functionRuntimeManager.setAssignment(assignment1);
    functionRuntimeManager.setAssignment(assignment2);
    membershipManager.checkFailures(functionMetaDataManager, functionRuntimeManager, schedulerManager);
    verify(schedulerManager, times(0)).schedule();
    verify(functionRuntimeManager, times(0)).removeAssignments(any());
    assertEquals(membershipManager.unsignedFunctionDurations.size(), 0);
}