Examples with FunctionMetaData org.apache.pulsar.functions.proto.Function.FunctionMetaData used on opensource projects

Example 6 with FunctionMetaData

use of org.apache.pulsar.functions.proto.Function.FunctionMetaData in project incubator-pulsar by apache.

the class FunctionActioner method stopFunction.

private void stopFunction(FunctionRuntimeInfo functionRuntimeInfo) {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    log.info("Stopping function {} - {}...", functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
    if (functionRuntimeInfo.getRuntimeSpawner() != null) {
        functionRuntimeInfo.getRuntimeSpawner().close();
        functionRuntimeInfo.setRuntimeSpawner(null);
    }
    // clean up function package
    File pkgDir = new File(workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData));
    if (pkgDir.exists()) {
        try {
            FileUtils.deleteDirectory(pkgDir);
        } catch (IOException e) {
            log.warn("Failed to delete package for function: {}", FunctionConfigUtils.getFullyQualifiedName(functionMetaData.getFunctionConfig()), e);
        }
    }
}

Example 7 with FunctionMetaData

use of org.apache.pulsar.functions.proto.Function.FunctionMetaData in project incubator-pulsar by apache.

the class FunctionActioner method startFunction.

private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Exception {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    log.info("Starting function {} - {} ...", functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
    File pkgDir = new File(workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData));
    pkgDir.mkdirs();
    int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId();
    File pkgFile = new File(pkgDir, new File(FunctionConfigUtils.getDownloadFileName(functionMetaData.getFunctionConfig())).getName());
    if (!pkgFile.exists()) {
        // download only when the package file doesn't exist
        File tempPkgFile;
        while (true) {
            tempPkgFile = new File(pkgDir, pkgFile.getName() + "." + instanceId + "." + UUID.randomUUID().toString());
            if (!tempPkgFile.exists() && tempPkgFile.createNewFile()) {
                break;
            }
        }
        try {
            log.info("Function package file {} will be downloaded from {}", tempPkgFile, functionMetaData.getPackageLocation());
            Utils.downloadFromBookkeeper(dlogNamespace, new FileOutputStream(tempPkgFile), functionMetaData.getPackageLocation().getPackagePath());
            // this ensures one instance will successfully download the package.
            try {
                Files.createLink(Paths.get(pkgFile.toURI()), Paths.get(tempPkgFile.toURI()));
                log.info("Function package file is linked from {} to {}", tempPkgFile, pkgFile);
            } catch (FileAlreadyExistsException faee) {
                // file already exists
                log.warn("Function package has been downloaded from {} and saved at {}", functionMetaData.getPackageLocation(), pkgFile);
            }
        } finally {
            tempPkgFile.delete();
        }
    }
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionConfig(functionMetaData.getFunctionConfig());
    // TODO: set correct function id and version when features implemented
    instanceConfig.setFunctionId(UUID.randomUUID().toString());
    instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
    instanceConfig.setInstanceId(String.valueOf(instanceId));
    instanceConfig.setMaxBufferedTuples(1024);
    RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, pkgFile.getAbsolutePath(), runtimeFactory, workerConfig.getInstanceLivenessCheckFreqMs());
    functionRuntimeInfo.setRuntimeSpawner(runtimeSpawner);
    runtimeSpawner.start();
}

Example 8 with FunctionMetaData

use of org.apache.pulsar.functions.proto.Function.FunctionMetaData in project incubator-pulsar by apache.

the class FunctionMetaDataManager method proccessDeregister.

@VisibleForTesting
synchronized void proccessDeregister(Request.ServiceRequest deregisterRequest) {
    FunctionMetaData deregisterRequestFs = deregisterRequest.getFunctionMetaData();
    String functionName = deregisterRequestFs.getFunctionConfig().getName();
    String tenant = deregisterRequestFs.getFunctionConfig().getTenant();
    String namespace = deregisterRequestFs.getFunctionConfig().getNamespace();
    boolean needsScheduling = false;
    log.debug("Process deregister request: {}", deregisterRequest);
    // Check if we still have this function. Maybe already deleted by someone else
    if (this.containsFunctionMetaData(deregisterRequestFs)) {
        // check if request is outdated
        if (!isRequestOutdated(deregisterRequest)) {
            this.functionMetaDataMap.get(tenant).get(namespace).remove(functionName);
            completeRequest(deregisterRequest, true);
            needsScheduling = true;
        } else {
            completeRequest(deregisterRequest, false, "Request ignored because it is out of date. Please try again.");
        }
    } else {
        // already deleted so  just complete request
        completeRequest(deregisterRequest, true);
    }
    if (!this.isInitializePhase() && needsScheduling) {
        this.schedulerManager.schedule();
    }
}

Example 9 with FunctionMetaData

use of org.apache.pulsar.functions.proto.Function.FunctionMetaData in project incubator-pulsar by apache.

the class FunctionMetaDataManager method deregisterFunction.

/**
 * Sends a deregister request to the FMT (Function Metadata Topic) for a function
 * @param tenant the tenant the function that needs to be deregistered belongs to
 * @param namespace the namespace the function that needs to be deregistered belongs to
 * @param functionName the name of the function
 * @return a completable future of when the deregister has been applied
 */
public synchronized CompletableFuture<RequestResult> deregisterFunction(String tenant, String namespace, String functionName) {
    FunctionMetaData functionMetaData = this.functionMetaDataMap.get(tenant).get(namespace).get(functionName);
    FunctionMetaData newFunctionMetaData = functionMetaData.toBuilder().setVersion(functionMetaData.getVersion() + 1).build();
    Request.ServiceRequest deregisterRequest = ServiceRequestUtils.getDeregisterRequest(this.workerConfig.getWorkerId(), newFunctionMetaData);
    return submit(deregisterRequest);
}

Example 10 with FunctionMetaData

use of org.apache.pulsar.functions.proto.Function.FunctionMetaData in project incubator-pulsar by apache.

the class FunctionMetaDataManager method processUpdate.

@VisibleForTesting
synchronized void processUpdate(Request.ServiceRequest updateRequest) {
    log.debug("Process update request: {}", updateRequest);
    FunctionMetaData updateRequestFs = updateRequest.getFunctionMetaData();
    boolean needsScheduling = false;
    // Worker doesn't know about the function so far
    if (!this.containsFunctionMetaData(updateRequestFs)) {
        // Since this is the first time worker has seen function, just put it into internal function metadata store
        setFunctionMetaData(updateRequestFs);
        needsScheduling = true;
        completeRequest(updateRequest, true);
    } else {
        // Check if request is outdated
        if (!isRequestOutdated(updateRequest)) {
            // update the function metadata
            setFunctionMetaData(updateRequestFs);
            needsScheduling = true;
            completeRequest(updateRequest, true);
        } else {
            completeRequest(updateRequest, false, "Request ignored because it is out of date. Please try again.");
        }
    }
    if (!this.isInitializePhase() && needsScheduling) {
        this.schedulerManager.schedule();
    }
}