Examples with Function org.apache.pulsar.functions.proto.Function used on opensource projects

Example 1 with Function

use of org.apache.pulsar.functions.proto.Function in project incubator-pulsar by apache.

the class MembershipManager method checkFailures.

public void checkFailures(FunctionMetaDataManager functionMetaDataManager, FunctionRuntimeManager functionRuntimeManager, SchedulerManager schedulerManager) {
    Set<String> currentMembership = this.getCurrentMembership().stream().map(entry -> entry.getWorkerId()).collect(Collectors.toSet());
    List<Function.FunctionMetaData> functionMetaDataList = functionMetaDataManager.getAllFunctionMetaData();
    Map<String, Function.FunctionMetaData> functionMetaDataMap = new HashMap<>();
    for (Function.FunctionMetaData entry : functionMetaDataList) {
        functionMetaDataMap.put(FunctionConfigUtils.getFullyQualifiedName(entry.getFunctionConfig()), entry);
    }
    Map<String, Map<String, Function.Assignment>> currentAssignments = functionRuntimeManager.getCurrentAssignments();
    Map<String, Function.Assignment> assignmentMap = new HashMap<>();
    for (Map<String, Function.Assignment> entry : currentAssignments.values()) {
        assignmentMap.putAll(entry);
    }
    long currentTimeMs = System.currentTimeMillis();
    // remove functions
    Iterator<Map.Entry<Function.Instance, Long>> it = unsignedFunctionDurations.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<Function.Instance, Long> entry = it.next();
        String fullyQualifiedFunctionName = FunctionConfigUtils.getFullyQualifiedName(entry.getKey().getFunctionMetaData().getFunctionConfig());
        String fullyQualifiedInstanceId = Utils.getFullyQualifiedInstanceId(entry.getKey());
        // remove functions that don't exist anymore
        if (!functionMetaDataMap.containsKey(fullyQualifiedFunctionName)) {
            it.remove();
        } else {
            // remove functions that have been scheduled
            Function.Assignment assignment = assignmentMap.get(fullyQualifiedInstanceId);
            if (assignment != null) {
                String assignedWorkerId = assignment.getWorkerId();
                // check if assigned to worker that has failed
                if (currentMembership.contains(assignedWorkerId)) {
                    it.remove();
                }
            }
        }
    }
    // check for function instances that haven't been assigned
    for (Function.FunctionMetaData functionMetaData : functionMetaDataList) {
        Collection<Function.Assignment> assignments = FunctionRuntimeManager.findFunctionAssignments(functionMetaData.getFunctionConfig().getTenant(), functionMetaData.getFunctionConfig().getNamespace(), functionMetaData.getFunctionConfig().getName(), currentAssignments);
        Set<Function.Instance> assignedInstances = assignments.stream().map(assignment -> assignment.getInstance()).collect(Collectors.toSet());
        Set<Function.Instance> instances = new HashSet<>(SchedulerManager.computeInstances(functionMetaData));
        for (Function.Instance instance : instances) {
            if (!assignedInstances.contains(instance)) {
                if (!this.unsignedFunctionDurations.containsKey(instance)) {
                    this.unsignedFunctionDurations.put(instance, currentTimeMs);
                }
            }
        }
    }
    // check failed nodes
    for (Map.Entry<String, Map<String, Function.Assignment>> entry : currentAssignments.entrySet()) {
        String workerId = entry.getKey();
        Map<String, Function.Assignment> assignmentEntries = entry.getValue();
        if (!currentMembership.contains(workerId)) {
            for (Function.Assignment assignmentEntry : assignmentEntries.values()) {
                Function.Instance instance = assignmentEntry.getInstance();
                if (!this.unsignedFunctionDurations.containsKey(instance)) {
                    this.unsignedFunctionDurations.put(instance, currentTimeMs);
                }
            }
        }
    }
    boolean triggerScheduler = false;
    // check unassigned
    Collection<Function.Instance> needSchedule = new LinkedList<>();
    Collection<Function.Assignment> needRemove = new LinkedList<>();
    for (Map.Entry<Function.Instance, Long> entry : this.unsignedFunctionDurations.entrySet()) {
        Function.Instance instance = entry.getKey();
        long unassignedDurationMs = entry.getValue();
        if (currentTimeMs - unassignedDurationMs > this.workerConfig.getRescheduleTimeoutMs()) {
            needSchedule.add(instance);
            // remove assignment from failed node
            Function.Assignment assignment = assignmentMap.get(Utils.getFullyQualifiedInstanceId(instance));
            if (assignment != null) {
                needRemove.add(assignment);
            }
            triggerScheduler = true;
        }
    }
    if (!needRemove.isEmpty()) {
        functionRuntimeManager.removeAssignments(needRemove);
    }
    if (triggerScheduler) {
        log.info("Functions that need scheduling/rescheduling: {}", needSchedule);
        schedulerManager.schedule();
    }
}

Example 2 with Function

use of org.apache.pulsar.functions.proto.Function in project incubator-pulsar by apache.

the class FunctionActioner method stopFunction.

private void stopFunction(FunctionRuntimeInfo functionRuntimeInfo) {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    log.info("Stopping function {} - {}...", functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
    if (functionRuntimeInfo.getRuntimeSpawner() != null) {
        functionRuntimeInfo.getRuntimeSpawner().close();
        functionRuntimeInfo.setRuntimeSpawner(null);
    }
    // clean up function package
    File pkgDir = new File(workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData));
    if (pkgDir.exists()) {
        try {
            FileUtils.deleteDirectory(pkgDir);
        } catch (IOException e) {
            log.warn("Failed to delete package for function: {}", FunctionConfigUtils.getFullyQualifiedName(functionMetaData.getFunctionConfig()), e);
        }
    }
}

Example 3 with Function

use of org.apache.pulsar.functions.proto.Function in project incubator-pulsar by apache.

the class FunctionActioner method startFunction.

private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Exception {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    log.info("Starting function {} - {} ...", functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
    File pkgDir = new File(workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData));
    pkgDir.mkdirs();
    int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId();
    File pkgFile = new File(pkgDir, new File(FunctionConfigUtils.getDownloadFileName(functionMetaData.getFunctionConfig())).getName());
    if (!pkgFile.exists()) {
        // download only when the package file doesn't exist
        File tempPkgFile;
        while (true) {
            tempPkgFile = new File(pkgDir, pkgFile.getName() + "." + instanceId + "." + UUID.randomUUID().toString());
            if (!tempPkgFile.exists() && tempPkgFile.createNewFile()) {
                break;
            }
        }
        try {
            log.info("Function package file {} will be downloaded from {}", tempPkgFile, functionMetaData.getPackageLocation());
            Utils.downloadFromBookkeeper(dlogNamespace, new FileOutputStream(tempPkgFile), functionMetaData.getPackageLocation().getPackagePath());
            // this ensures one instance will successfully download the package.
            try {
                Files.createLink(Paths.get(pkgFile.toURI()), Paths.get(tempPkgFile.toURI()));
                log.info("Function package file is linked from {} to {}", tempPkgFile, pkgFile);
            } catch (FileAlreadyExistsException faee) {
                // file already exists
                log.warn("Function package has been downloaded from {} and saved at {}", functionMetaData.getPackageLocation(), pkgFile);
            }
        } finally {
            tempPkgFile.delete();
        }
    }
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionConfig(functionMetaData.getFunctionConfig());
    // TODO: set correct function id and version when features implemented
    instanceConfig.setFunctionId(UUID.randomUUID().toString());
    instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
    instanceConfig.setInstanceId(String.valueOf(instanceId));
    instanceConfig.setMaxBufferedTuples(1024);
    RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, pkgFile.getAbsolutePath(), runtimeFactory, workerConfig.getInstanceLivenessCheckFreqMs());
    functionRuntimeInfo.setRuntimeSpawner(runtimeSpawner);
    runtimeSpawner.start();
}

Example 4 with Function

use of org.apache.pulsar.functions.proto.Function in project incubator-pulsar by apache.

the class SchedulerManager method invokeScheduler.

private void invokeScheduler() {
    List<String> currentMembership = this.membershipManager.getCurrentMembership().stream().map(workerInfo -> workerInfo.getWorkerId()).collect(Collectors.toList());
    List<FunctionMetaData> allFunctions = this.functionMetaDataManager.getAllFunctionMetaData();
    Map<String, Function.Instance> allInstances = computeAllInstances(allFunctions);
    Map<String, Map<String, Assignment>> workerIdToAssignments = this.functionRuntimeManager.getCurrentAssignments();
    // delete assignments of functions and instances that don't exist anymore
    Iterator<Map.Entry<String, Map<String, Assignment>>> it = workerIdToAssignments.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<String, Map<String, Assignment>> workerIdToAssignmentEntry = it.next();
        Map<String, Assignment> functionMap = workerIdToAssignmentEntry.getValue();
        // remove instances that don't exist anymore
        functionMap.entrySet().removeIf(entry -> {
            String fullyQualifiedInstanceId = entry.getKey();
            return !allInstances.containsKey(fullyQualifiedInstanceId);
        });
        // update assignment instances in case attributes of a function gets updated
        for (Map.Entry<String, Assignment> entry : functionMap.entrySet()) {
            String fullyQualifiedInstanceId = entry.getKey();
            Assignment assignment = entry.getValue();
            Function.Instance instance = allInstances.get(fullyQualifiedInstanceId);
            if (!assignment.getInstance().equals(instance)) {
                functionMap.put(fullyQualifiedInstanceId, assignment.toBuilder().setInstance(instance).build());
            }
        }
        if (functionMap.isEmpty()) {
            it.remove();
        }
    }
    List<Assignment> currentAssignments = workerIdToAssignments.entrySet().stream().flatMap(stringMapEntry -> stringMapEntry.getValue().values().stream()).collect(Collectors.toList());
    List<Function.Instance> needsAssignment = this.getUnassignedFunctionInstances(workerIdToAssignments, allInstances);
    List<Assignment> assignments = this.scheduler.schedule(needsAssignment, currentAssignments, currentMembership);
    log.debug("New assignments computed: {}", assignments);
    long assignmentVersion = this.functionRuntimeManager.getCurrentAssignmentVersion() + 1;
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.newBuilder().setVersion(assignmentVersion).addAllAssignments(assignments).build();
    CompletableFuture<MessageId> messageIdCompletableFuture = producer.sendAsync(assignmentsUpdate.toByteArray());
    try {
        messageIdCompletableFuture.get();
    } catch (InterruptedException | ExecutionException e) {
        log.error("Failed to send assignment update", e);
        throw new RuntimeException(e);
    }
    // wait for assignment update to go throw the pipeline
    int retries = 0;
    while (this.functionRuntimeManager.getCurrentAssignmentVersion() < assignmentVersion) {
        if (retries >= this.workerConfig.getAssignmentWriteMaxRetries()) {
            log.warn("Max number of retries reached for waiting for assignment to propagate. Will continue now.");
            break;
        }
        log.info("Waiting for assignments to propagate...");
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        retries++;
    }
}