Examples with Assignment org.apache.pulsar.functions.proto.Function.Assignment used on opensource projects

Example 1 with Assignment

use of org.apache.pulsar.functions.proto.Function.Assignment in project incubator-pulsar by apache.

the class FunctionRuntimeManager method findAssignment.

private Assignment findAssignment(String tenant, String namespace, String functionName, int instanceId) {
    String fullyQualifiedInstanceId = Utils.getFullyQualifiedInstanceId(tenant, namespace, functionName, instanceId);
    for (Map.Entry<String, Map<String, Assignment>> entry : this.workerIdToAssignments.entrySet()) {
        Map<String, Assignment> assignmentMap = entry.getValue();
        Assignment existingAssignment = assignmentMap.get(fullyQualifiedInstanceId);
        if (existingAssignment != null) {
            return existingAssignment;
        }
    }
    return null;
}

Example 2 with Assignment

use of org.apache.pulsar.functions.proto.Function.Assignment in project incubator-pulsar by apache.

the class RoundRobinScheduler method schedule.

@Override
public List<Assignment> schedule(List<Instance> unassignedFunctionInstances, List<Assignment> currentAssignments, List<String> workers) {
    Map<String, List<Assignment>> workerIdToAssignment = new HashMap<>();
    for (String workerId : workers) {
        workerIdToAssignment.put(workerId, new LinkedList<>());
    }
    for (Assignment existingAssignment : currentAssignments) {
        workerIdToAssignment.get(existingAssignment.getWorkerId()).add(existingAssignment);
    }
    for (Instance unassignedFunctionInstance : unassignedFunctionInstances) {
        String workerId = findNextWorker(workerIdToAssignment);
        Assignment newAssignment = Assignment.newBuilder().setInstance(unassignedFunctionInstance).setWorkerId(workerId).build();
        workerIdToAssignment.get(workerId).add(newAssignment);
    }
    List<Assignment> assignments = workerIdToAssignment.entrySet().stream().flatMap(entry -> entry.getValue().stream()).collect(Collectors.toList());
    return assignments;
}

Example 3 with Assignment

use of org.apache.pulsar.functions.proto.Function.Assignment in project incubator-pulsar by apache.

the class FunctionRuntimeManager method getAllFunctionStatus.

/**
 * Get statuses of all function instances.
 * @param tenant the tenant the function belongs to
 * @param namespace the namespace the function belongs to
 * @param functionName the function name
 * @return a list of function statuses
 */
public InstanceCommunication.FunctionStatusList getAllFunctionStatus(String tenant, String namespace, String functionName) {
    Collection<Assignment> assignments = this.findFunctionAssignments(tenant, namespace, functionName);
    InstanceCommunication.FunctionStatusList.Builder functionStatusListBuilder = InstanceCommunication.FunctionStatusList.newBuilder();
    if (assignments.isEmpty()) {
        return functionStatusListBuilder.build();
    }
    for (Assignment assignment : assignments) {
        InstanceCommunication.FunctionStatus functionStatus = this.getFunctionInstanceStatus(assignment.getInstance().getFunctionMetaData().getFunctionConfig().getTenant(), assignment.getInstance().getFunctionMetaData().getFunctionConfig().getNamespace(), assignment.getInstance().getFunctionMetaData().getFunctionConfig().getName(), assignment.getInstance().getInstanceId());
        functionStatusListBuilder.addFunctionStatusList(functionStatus);
    }
    return functionStatusListBuilder.build();
}

Example 4 with Assignment

use of org.apache.pulsar.functions.proto.Function.Assignment in project incubator-pulsar by apache.

the class FunctionRuntimeManager method getFunctionInstanceStatus.

/**
 * Get status of a function instance.  If this worker is not running the function instance,
 * @param tenant the tenant the function belongs to
 * @param namespace the namespace the function belongs to
 * @param functionName the function name
 * @param instanceId the function instance id
 * @return the function status
 */
public InstanceCommunication.FunctionStatus getFunctionInstanceStatus(String tenant, String namespace, String functionName, int instanceId) {
    String workerId = this.workerConfig.getWorkerId();
    Assignment assignment = this.findAssignment(tenant, namespace, functionName, instanceId);
    if (assignment == null) {
        InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
        functionStatusBuilder.setRunning(false);
        functionStatusBuilder.setFailureException("Function has not been scheduled");
        return functionStatusBuilder.build();
    }
    InstanceCommunication.FunctionStatus functionStatus = null;
    // If I am running worker
    if (assignment.getWorkerId().equals(workerId)) {
        FunctionRuntimeInfo functionRuntimeInfo = this.getFunctionRuntimeInfo(Utils.getFullyQualifiedInstanceId(assignment.getInstance()));
        RuntimeSpawner runtimeSpawner = functionRuntimeInfo.getRuntimeSpawner();
        if (runtimeSpawner != null) {
            try {
                functionStatus = functionRuntimeInfo.getRuntimeSpawner().getFunctionStatus().get();
            } catch (InterruptedException | ExecutionException e) {
                throw new RuntimeException(e);
            }
        } else {
            InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
            functionStatusBuilder.setRunning(false);
            functionStatusBuilder.setInstanceId(String.valueOf(instanceId));
            if (functionRuntimeInfo.getStartupException() != null) {
                functionStatusBuilder.setFailureException(functionRuntimeInfo.getStartupException().getMessage());
            }
            functionStatus = functionStatusBuilder.build();
        }
    } else {
        // query other worker
        List<MembershipManager.WorkerInfo> workerInfoList = this.membershipManager.getCurrentMembership();
        MembershipManager.WorkerInfo workerInfo = null;
        for (MembershipManager.WorkerInfo entry : workerInfoList) {
            if (assignment.getWorkerId().equals(entry.getWorkerId())) {
                workerInfo = entry;
            }
        }
        if (workerInfo == null) {
            InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
            functionStatusBuilder.setRunning(false);
            functionStatusBuilder.setInstanceId(String.valueOf(instanceId));
            functionStatusBuilder.setFailureException("Function has not been scheduled");
            return functionStatusBuilder.build();
        }
        Client client = ClientBuilder.newClient();
        // TODO: implement authentication/authorization
        String jsonResponse = client.target(String.format("http://%s:%d/admin/functions/%s/%s/%s/%d/status", workerInfo.getWorkerHostname(), workerInfo.getPort(), tenant, namespace, functionName, instanceId)).request(MediaType.TEXT_PLAIN).get(String.class);
        InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
        try {
            org.apache.pulsar.functions.utils.Utils.mergeJson(jsonResponse, functionStatusBuilder);
        } catch (IOException e) {
            log.warn("Got invalid function status response from {}", workerInfo, e);
            throw new RuntimeException(e);
        }
        functionStatus = functionStatusBuilder.build();
    }
    return functionStatus;
}

Example 5 with Assignment

use of org.apache.pulsar.functions.proto.Function.Assignment in project incubator-pulsar by apache.

the class FunctionRuntimeManager method processAssignmentUpdate.

/**
 * Process an assignment update from the assignment topic
 * @param messageId the message id of the update assignment
 * @param assignmentsUpdate the assignment update
 */
public synchronized void processAssignmentUpdate(MessageId messageId, AssignmentsUpdate assignmentsUpdate) {
    if (assignmentsUpdate.getVersion() > this.currentAssignmentVersion) {
        Map<String, Assignment> assignmentMap = new HashMap<>();
        for (Assignment assignment : assignmentsUpdate.getAssignmentsList()) {
            assignmentMap.put(Utils.getFullyQualifiedInstanceId(assignment.getInstance()), assignment);
        }
        Map<String, Assignment> existingAssignmentMap = new HashMap<>();
        for (Map<String, Assignment> entry : this.workerIdToAssignments.values()) {
            existingAssignmentMap.putAll(entry);
        }
        Map<String, Assignment> assignmentsToAdd = diff(assignmentMap, existingAssignmentMap);
        Map<String, Assignment> assignmentsToDelete = diff(existingAssignmentMap, assignmentMap);
        Map<String, Assignment> existingAssignments = inCommon(assignmentMap, existingAssignmentMap);
        // functions to add
        for (Map.Entry<String, Assignment> assignmentEntry : assignmentsToAdd.entrySet()) {
            String fullyQualifiedInstanceId = assignmentEntry.getKey();
            Assignment assignment = assignmentEntry.getValue();
            // add new function
            this.setAssignment(assignment);
            // Assigned to me
            if (assignment.getWorkerId().equals(workerConfig.getWorkerId())) {
                if (!this.functionRuntimeInfoMap.containsKey(fullyQualifiedInstanceId)) {
                    this.setFunctionRuntimeInfo(fullyQualifiedInstanceId, new FunctionRuntimeInfo().setFunctionInstance(assignment.getInstance()));
                } else {
                    // Somehow this function is already started
                    log.warn("Function {} already running. Going to restart function.", this.functionRuntimeInfoMap.get(fullyQualifiedInstanceId));
                    this.insertStopAction(this.functionRuntimeInfoMap.get(fullyQualifiedInstanceId));
                }
                FunctionRuntimeInfo functionRuntimeInfo = this.functionRuntimeInfoMap.get(fullyQualifiedInstanceId);
                this.insertStartAction(functionRuntimeInfo);
            }
        }
        // functions to delete
        for (Map.Entry<String, Assignment> assignmentEntry : assignmentsToDelete.entrySet()) {
            String fullyQualifiedInstanceId = assignmentEntry.getKey();
            Assignment assignment = assignmentEntry.getValue();
            FunctionRuntimeInfo functionRuntimeInfo = this.functionRuntimeInfoMap.get(fullyQualifiedInstanceId);
            if (functionRuntimeInfo != null) {
                this.insertStopAction(functionRuntimeInfo);
                this.deleteFunctionRuntimeInfo(fullyQualifiedInstanceId);
            }
            this.deleteAssignment(assignment);
        }
        // functions to update
        for (Map.Entry<String, Assignment> assignmentEntry : existingAssignments.entrySet()) {
            String fullyQualifiedInstanceId = assignmentEntry.getKey();
            Assignment assignment = assignmentEntry.getValue();
            Assignment existingAssignment = this.findAssignment(assignment);
            // potential updates need to happen
            if (!existingAssignment.equals(assignment)) {
                FunctionRuntimeInfo functionRuntimeInfo = this.functionRuntimeInfoMap.get(fullyQualifiedInstanceId);
                // stop function
                if (functionRuntimeInfo != null) {
                    this.insertStopAction(functionRuntimeInfo);
                }
                // still assigned to me, need to restart
                if (assignment.getWorkerId().equals(this.workerConfig.getWorkerId())) {
                    // start again
                    FunctionRuntimeInfo newFunctionRuntimeInfo = new FunctionRuntimeInfo();
                    newFunctionRuntimeInfo.setFunctionInstance(assignment.getInstance());
                    this.insertStartAction(newFunctionRuntimeInfo);
                    this.setFunctionRuntimeInfo(fullyQualifiedInstanceId, newFunctionRuntimeInfo);
                    this.setAssignment(assignment);
                }
            }
        }
        // set as current assignment
        this.currentAssignmentVersion = assignmentsUpdate.getVersion();
    } else {
        log.debug("Received out of date assignment update: {}", assignmentsUpdate);
    }
}