Examples with WorkerInfo org.apache.pulsar.common.functions.WorkerInfo used on opensource projects

Example 26 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class PulsarWorkerRebalanceDrainTest method testRebalance.

private void testRebalance() throws Exception {
    // Add some workers; then call rebalance, and check that functions were assigned to all of the workers.
    allocateFunctions("testRebalanceAddWorkers", "test-rebalance");
    if (log.isDebugEnabled()) {
        this.showWorkerStatus("testRebalanceAddWorkers after allocating functions");
    }
    WorkerInfo oldClusterLeaderInfo = getClusterLeader();
    log.info("Cluster leader before adding more workers is: {}", oldClusterLeaderInfo);
    List<Map<String, Collection<String>>> startFinfos = getFunctionAssignments();
    int startFuncCount = getFuncAssignmentsCount(startFinfos);
    log.info("testRebalanceAddWorkers: got info about {} workers with {} functions before creating new workers", startFinfos.size(), startFuncCount);
    // Check that there are NumFunctionsAssignedOnEachWorker functions assigned to each worker,
    // since the assignment is round-robin by default.
    assertEquals(getMinFuncAssignmentOnAnyWorker(startFinfos), NumFunctionsAssignedOnEachWorker);
    // Add a few more workers, to test rebalance
    int initialNumWorkers = pulsarCluster.getAlWorkers().size();
    final int numWorkersToAdd = 2;
    log.info("testRebalanceAddWorkers: cluster has {} FunctionWorkers; going to set up {} more", pulsarCluster.getAlWorkers().size(), numWorkersToAdd);
    pulsarCluster.setupFunctionWorkers(randomName(5), functionRuntimeType, numWorkersToAdd);
    assertEquals(pulsarCluster.getAlWorkers().size(), initialNumWorkers + numWorkersToAdd);
    log.info("testRebalanceAddWorkers: got a total of {} function workers, of type {}", pulsarCluster.getAlWorkers().size(), functionRuntimeType);
    this.showWorkerStatus("testRebalanceAddWorkers status after adding more workers");
    WorkerInfo newClusterLeaderInfo = getClusterLeader();
    log.info("Cluster leader after adding {} workers is: {}", numWorkersToAdd, newClusterLeaderInfo);
    // Leadership should not have changed.
    assertTrue(oldClusterLeaderInfo.getWorkerId().compareTo(newClusterLeaderInfo.getWorkerId()) == 0);
    this.showWorkerStatus("testRebalanceAddWorkers after adding more workers");
    // Rebalance.
    callRebalance();
    this.showWorkerStatus("testRebalanceAddWorkers after rebalance");
    List<Map<String, Collection<String>>> endFinfos = getFunctionAssignments();
    int endFuncCount = getFuncAssignmentsCount(endFinfos);
    log.info("testRebalanceAddWorkers: got info about {} workers with {} functions after rebalance", endFinfos.size(), endFuncCount);
    assertEquals(endFinfos.size() - startFinfos.size(), numWorkersToAdd);
    assertEquals(startFuncCount, endFuncCount);
    // Since scheduling is round-robin (default), we expect the minimum number of  function assignments
    // on any worker to be the floor of the average number of functions per worker.
    int minFuncsPerWorker = (int) Math.floor(endFuncCount / endFinfos.size());
    assertEquals(getMinFuncAssignmentOnAnyWorker(endFinfos), minFuncsPerWorker);
    // Since we increased the number of workers, the minFuncsPerWorker should evaluate to some
    // value less than the erstwhile NumFunctionsAssignedOnEachWorker.
    assertTrue(minFuncsPerWorker < NumFunctionsAssignedOnEachWorker);
}

Example 27 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class PulsarWorkerRebalanceDrainTest method getClusterLeader.

private WorkerInfo getClusterLeader() throws Exception {
    ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions-worker", "get-cluster-leader");
    List<WorkerInfo> winfos = workerInfoDecode(result.getStdout());
    assertEquals(winfos.size(), 1);
    return workerInfoDecode(result.getStdout()).get(0);
}

Example 28 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project pulsar by yahoo.

the class PulsarWorkerRebalanceDrainTest method testDrain.

private void testDrain() throws Exception {
    allocateFunctions("testDrain", "test-drain");
    val startFinfos = getFunctionAssignments();
    int startFuncCount = getFuncAssignmentsCount(startFinfos);
    log.info("testDrain: got info about {} workers with {} functions before drain", startFinfos.size(), startFuncCount);
    if (log.isDebugEnabled()) {
        this.showWorkerStatus("testDrain after allocating functions");
    }
    WorkerInfo clusterLeaderInfo = getClusterLeader();
    // Drain
    callDrain(clusterLeaderInfo.getWorkerId());
    if (log.isDebugEnabled()) {
        this.showWorkerStatus("testDrain after drain");
    }
    val endFinfos = getFunctionAssignments();
    int endFuncCount = getFuncAssignmentsCount(endFinfos);
    log.info("testDrain: got info about {} workers with {} functions after drain", endFinfos.size(), endFuncCount);
    assertTrue(startFinfos.size() > endFinfos.size());
    assertEquals(startFuncCount, endFuncCount);
    // Since default scheduling is round-robin, we expect the minimum number of  function assignments
    // on any worker to be the floor of the average number of functions per worker.
    final int minFuncsPerWorker = (int) Math.floor(endFuncCount / endFinfos.size());
    assertEquals(getMinFuncAssignmentOnAnyWorker(endFinfos), minFuncsPerWorker);
}

Example 29 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project incubator-pulsar by apache.

the class WorkerImpl method getClusterLeaderAsync.

@Override
public CompletableFuture<WorkerInfo> getClusterLeaderAsync() {
    WebTarget path = worker.path("cluster").path("leader");
    final CompletableFuture<WorkerInfo> future = new CompletableFuture<>();
    asyncGetRequest(path, new InvocationCallback<Response>() {

        @Override
        public void completed(Response response) {
            if (response.getStatus() != Response.Status.OK.getStatusCode()) {
                future.completeExceptionally(new ClientErrorException(response));
            } else {
                future.complete(response.readEntity(new GenericType<WorkerInfo>() {
                }));
            }
        }

        @Override
        public void failed(Throwable throwable) {
            future.completeExceptionally(getApiException(throwable.getCause()));
        }
    });
    return future;
}

Example 30 with WorkerInfo

use of org.apache.pulsar.common.functions.WorkerInfo in project incubator-pulsar by apache.

the class FunctionRuntimeManagerTest method testThreadFunctionInstancesRestart.

@Test
public void testThreadFunctionInstancesRestart() throws Exception {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
    workerConfig.setFunctionRuntimeFactoryConfigs(ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("test"), Map.class));
    workerConfig.setPulsarServiceUrl(PULSAR_SERVICE_URL);
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setFunctionAssignmentTopicName("assignments");
    PulsarWorkerService workerService = mock(PulsarWorkerService.class);
    // mock pulsarAdmin sources sinks functions
    PulsarAdmin pulsarAdmin = mock(PulsarAdmin.class);
    Sources sources = mock(Sources.class);
    doNothing().when(sources).restartSource(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    doReturn(sources).when(pulsarAdmin).sources();
    Sinks sinks = mock(Sinks.class);
    doReturn(sinks).when(pulsarAdmin).sinks();
    Functions functions = mock(Functions.class);
    doNothing().when(functions).restartFunction(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    doReturn(functions).when(pulsarAdmin).functions();
    doReturn(pulsarAdmin).when(workerService).getFunctionAdmin();
    try (final MockedStatic<RuntimeFactory> runtimeFactoryMockedStatic = Mockito.mockStatic(RuntimeFactory.class)) {
        mockRuntimeFactory(runtimeFactoryMockedStatic);
        List<WorkerInfo> workerInfos = new LinkedList<>();
        workerInfos.add(WorkerInfo.of("worker-1", "localhost", 0));
        workerInfos.add(WorkerInfo.of("worker-2", "localhost", 0));
        MembershipManager membershipManager = mock(MembershipManager.class);
        doReturn(workerInfos).when(membershipManager).getCurrentMembership();
        // build three types of FunctionMetaData
        Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("function").setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)).build();
        Function.FunctionMetaData source = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("source").setComponentType(Function.FunctionDetails.ComponentType.SOURCE)).build();
        Function.FunctionMetaData sink = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("sink").setComponentType(Function.FunctionDetails.ComponentType.SINK)).build();
        FunctionRuntimeManager functionRuntimeManager = spy(new FunctionRuntimeManager(workerConfig, workerService, mock(Namespace.class), membershipManager, mock(ConnectorsManager.class), mock(FunctionsManager.class), mock(FunctionMetaDataManager.class), mock(WorkerStatsManager.class), mock(ErrorNotifier.class)));
        // verify restart function/source/sink using different assignment
        verifyRestart(functionRuntimeManager, function, "worker-1", false, false);
        verifyRestart(functionRuntimeManager, function, "worker-2", false, true);
        verifyRestart(functionRuntimeManager, source, "worker-1", false, false);
        verifyRestart(functionRuntimeManager, source, "worker-2", false, true);
        verifyRestart(functionRuntimeManager, sink, "worker-1", false, false);
        verifyRestart(functionRuntimeManager, sink, "worker-2", false, true);
    }
}