Search in sources :

Example 6 with Assignment

use of org.apache.pulsar.functions.proto.Function.Assignment in project incubator-pulsar by apache.

the class SchedulerManager method invokeScheduler.

private void invokeScheduler() {
    List<String> currentMembership = this.membershipManager.getCurrentMembership().stream().map(workerInfo -> workerInfo.getWorkerId()).collect(Collectors.toList());
    List<FunctionMetaData> allFunctions = this.functionMetaDataManager.getAllFunctionMetaData();
    Map<String, Function.Instance> allInstances = computeAllInstances(allFunctions);
    Map<String, Map<String, Assignment>> workerIdToAssignments = this.functionRuntimeManager.getCurrentAssignments();
    // delete assignments of functions and instances that don't exist anymore
    Iterator<Map.Entry<String, Map<String, Assignment>>> it = workerIdToAssignments.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<String, Map<String, Assignment>> workerIdToAssignmentEntry = it.next();
        Map<String, Assignment> functionMap = workerIdToAssignmentEntry.getValue();
        // remove instances that don't exist anymore
        functionMap.entrySet().removeIf(entry -> {
            String fullyQualifiedInstanceId = entry.getKey();
            return !allInstances.containsKey(fullyQualifiedInstanceId);
        });
        // update assignment instances in case attributes of a function gets updated
        for (Map.Entry<String, Assignment> entry : functionMap.entrySet()) {
            String fullyQualifiedInstanceId = entry.getKey();
            Assignment assignment = entry.getValue();
            Function.Instance instance = allInstances.get(fullyQualifiedInstanceId);
            if (!assignment.getInstance().equals(instance)) {
                functionMap.put(fullyQualifiedInstanceId, assignment.toBuilder().setInstance(instance).build());
            }
        }
        if (functionMap.isEmpty()) {
            it.remove();
        }
    }
    List<Assignment> currentAssignments = workerIdToAssignments.entrySet().stream().flatMap(stringMapEntry -> stringMapEntry.getValue().values().stream()).collect(Collectors.toList());
    List<Function.Instance> needsAssignment = this.getUnassignedFunctionInstances(workerIdToAssignments, allInstances);
    List<Assignment> assignments = this.scheduler.schedule(needsAssignment, currentAssignments, currentMembership);
    log.debug("New assignments computed: {}", assignments);
    long assignmentVersion = this.functionRuntimeManager.getCurrentAssignmentVersion() + 1;
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.newBuilder().setVersion(assignmentVersion).addAllAssignments(assignments).build();
    CompletableFuture<MessageId> messageIdCompletableFuture = producer.sendAsync(assignmentsUpdate.toByteArray());
    try {
        messageIdCompletableFuture.get();
    } catch (InterruptedException | ExecutionException e) {
        log.error("Failed to send assignment update", e);
        throw new RuntimeException(e);
    }
    // wait for assignment update to go throw the pipeline
    int retries = 0;
    while (this.functionRuntimeManager.getCurrentAssignmentVersion() < assignmentVersion) {
        if (retries >= this.workerConfig.getAssignmentWriteMaxRetries()) {
            log.warn("Max number of retries reached for waiting for assignment to propagate. Will continue now.");
            break;
        }
        log.info("Waiting for assignments to propagate...");
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        retries++;
    }
}
Also used : Setter(lombok.Setter) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) Request(org.apache.pulsar.functions.proto.Request) Producer(org.apache.pulsar.client.api.Producer) FunctionMetaData(org.apache.pulsar.functions.proto.Function.FunctionMetaData) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Future(java.util.concurrent.Future) Map(java.util.Map) PulsarClient(org.apache.pulsar.client.api.PulsarClient) LinkedList(java.util.LinkedList) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ExecutorService(java.util.concurrent.ExecutorService) Function(org.apache.pulsar.functions.proto.Function) Iterator(java.util.Iterator) CompressionType(org.apache.pulsar.client.api.CompressionType) Reflections(org.apache.pulsar.functions.utils.Reflections) Assignment(org.apache.pulsar.functions.proto.Function.Assignment) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) MessageId(org.apache.pulsar.client.api.MessageId) IScheduler(org.apache.pulsar.functions.worker.scheduler.IScheduler) FunctionMetaData(org.apache.pulsar.functions.proto.Function.FunctionMetaData) Assignment(org.apache.pulsar.functions.proto.Function.Assignment) Function(org.apache.pulsar.functions.proto.Function) ExecutionException(java.util.concurrent.ExecutionException) Request(org.apache.pulsar.functions.proto.Request) HashMap(java.util.HashMap) Map(java.util.Map) MessageId(org.apache.pulsar.client.api.MessageId)

Aggregations

Assignment (org.apache.pulsar.functions.proto.Function.Assignment)6 HashMap (java.util.HashMap)4 Map (java.util.Map)4 LinkedList (java.util.LinkedList)2 List (java.util.List)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 ExecutionException (java.util.concurrent.ExecutionException)2 Collectors (java.util.stream.Collectors)2 PulsarClient (org.apache.pulsar.client.api.PulsarClient)2 InstanceCommunication (org.apache.pulsar.functions.proto.InstanceCommunication)2 IOException (java.io.IOException)1 Iterator (java.util.Iterator)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 LinkedBlockingQueue (java.util.concurrent.LinkedBlockingQueue)1 ThreadPoolExecutor (java.util.concurrent.ThreadPoolExecutor)1 TimeUnit (java.util.concurrent.TimeUnit)1 Client (javax.ws.rs.client.Client)1 Setter (lombok.Setter)1