Examples with RuntimeSpawner org.apache.pulsar.functions.runtime.RuntimeSpawner used on opensource projects

Example 1 with RuntimeSpawner

use of org.apache.pulsar.functions.runtime.RuntimeSpawner in project incubator-pulsar by apache.

the class FunctionsStatsGenerator method generate.

public static void generate(WorkerService workerService, String cluster, SimpleTextOutputStream out) {
    if (workerService != null) {
        Map<String, FunctionRuntimeInfo> functionRuntimes = workerService.getFunctionRuntimeManager().getFunctionRuntimeInfos();
        for (Map.Entry<String, FunctionRuntimeInfo> entry : functionRuntimes.entrySet()) {
            String fullyQualifiedInstanceName = entry.getKey();
            FunctionRuntimeInfo functionRuntimeInfo = entry.getValue();
            RuntimeSpawner functionRuntimeSpawner = functionRuntimeInfo.getRuntimeSpawner();
            if (functionRuntimeSpawner != null) {
                Runtime functionRuntime = functionRuntimeSpawner.getRuntime();
                if (functionRuntime != null) {
                    try {
                        InstanceCommunication.MetricsData metrics = functionRuntime.getAndResetMetrics().get();
                        for (Map.Entry<String, InstanceCommunication.MetricsData.DataDigest> metricsEntry : metrics.getMetricsMap().entrySet()) {
                            String metricName = metricsEntry.getKey();
                            InstanceCommunication.MetricsData.DataDigest dataDigest = metricsEntry.getValue();
                            String tenant = functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionConfig().getTenant();
                            String namespace = functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionConfig().getNamespace();
                            String name = functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionConfig().getName();
                            int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId();
                            String qualifiedNamespace = String.format("%s/%s", tenant, namespace);
                            metric(out, cluster, qualifiedNamespace, name, String.format("pulsar_function%scount", metricName), instanceId, dataDigest.getCount());
                            metric(out, cluster, qualifiedNamespace, name, String.format("pulsar_function%smax", metricName), instanceId, dataDigest.getMax());
                            metric(out, cluster, qualifiedNamespace, name, String.format("pulsar_function%smin", metricName), instanceId, dataDigest.getMin());
                            metric(out, cluster, qualifiedNamespace, name, String.format("pulsar_function%ssum", metricName), instanceId, dataDigest.getSum());
                        }
                    } catch (InterruptedException | ExecutionException e) {
                        log.warn("Failed to collect metrics for function instance {}", fullyQualifiedInstanceName, e);
                    }
                }
            }
        }
    }
}

Example 2 with RuntimeSpawner

use of org.apache.pulsar.functions.runtime.RuntimeSpawner in project incubator-pulsar by apache.

the class FunctionActioner method startFunction.

private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Exception {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    log.info("Starting function {} - {} ...", functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
    File pkgDir = new File(workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData));
    pkgDir.mkdirs();
    int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId();
    File pkgFile = new File(pkgDir, new File(FunctionConfigUtils.getDownloadFileName(functionMetaData.getFunctionConfig())).getName());
    if (!pkgFile.exists()) {
        // download only when the package file doesn't exist
        File tempPkgFile;
        while (true) {
            tempPkgFile = new File(pkgDir, pkgFile.getName() + "." + instanceId + "." + UUID.randomUUID().toString());
            if (!tempPkgFile.exists() && tempPkgFile.createNewFile()) {
                break;
            }
        }
        try {
            log.info("Function package file {} will be downloaded from {}", tempPkgFile, functionMetaData.getPackageLocation());
            Utils.downloadFromBookkeeper(dlogNamespace, new FileOutputStream(tempPkgFile), functionMetaData.getPackageLocation().getPackagePath());
            // this ensures one instance will successfully download the package.
            try {
                Files.createLink(Paths.get(pkgFile.toURI()), Paths.get(tempPkgFile.toURI()));
                log.info("Function package file is linked from {} to {}", tempPkgFile, pkgFile);
            } catch (FileAlreadyExistsException faee) {
                // file already exists
                log.warn("Function package has been downloaded from {} and saved at {}", functionMetaData.getPackageLocation(), pkgFile);
            }
        } finally {
            tempPkgFile.delete();
        }
    }
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionConfig(functionMetaData.getFunctionConfig());
    // TODO: set correct function id and version when features implemented
    instanceConfig.setFunctionId(UUID.randomUUID().toString());
    instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
    instanceConfig.setInstanceId(String.valueOf(instanceId));
    instanceConfig.setMaxBufferedTuples(1024);
    RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, pkgFile.getAbsolutePath(), runtimeFactory, workerConfig.getInstanceLivenessCheckFreqMs());
    functionRuntimeInfo.setRuntimeSpawner(runtimeSpawner);
    runtimeSpawner.start();
}

Example 3 with RuntimeSpawner

use of org.apache.pulsar.functions.runtime.RuntimeSpawner in project incubator-pulsar by apache.

the class FunctionRuntimeManager method getFunctionInstanceStatus.

/**
 * Get status of a function instance.  If this worker is not running the function instance,
 * @param tenant the tenant the function belongs to
 * @param namespace the namespace the function belongs to
 * @param functionName the function name
 * @param instanceId the function instance id
 * @return the function status
 */
public InstanceCommunication.FunctionStatus getFunctionInstanceStatus(String tenant, String namespace, String functionName, int instanceId) {
    String workerId = this.workerConfig.getWorkerId();
    Assignment assignment = this.findAssignment(tenant, namespace, functionName, instanceId);
    if (assignment == null) {
        InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
        functionStatusBuilder.setRunning(false);
        functionStatusBuilder.setFailureException("Function has not been scheduled");
        return functionStatusBuilder.build();
    }
    InstanceCommunication.FunctionStatus functionStatus = null;
    // If I am running worker
    if (assignment.getWorkerId().equals(workerId)) {
        FunctionRuntimeInfo functionRuntimeInfo = this.getFunctionRuntimeInfo(Utils.getFullyQualifiedInstanceId(assignment.getInstance()));
        RuntimeSpawner runtimeSpawner = functionRuntimeInfo.getRuntimeSpawner();
        if (runtimeSpawner != null) {
            try {
                functionStatus = functionRuntimeInfo.getRuntimeSpawner().getFunctionStatus().get();
            } catch (InterruptedException | ExecutionException e) {
                throw new RuntimeException(e);
            }
        } else {
            InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
            functionStatusBuilder.setRunning(false);
            functionStatusBuilder.setInstanceId(String.valueOf(instanceId));
            if (functionRuntimeInfo.getStartupException() != null) {
                functionStatusBuilder.setFailureException(functionRuntimeInfo.getStartupException().getMessage());
            }
            functionStatus = functionStatusBuilder.build();
        }
    } else {
        // query other worker
        List<MembershipManager.WorkerInfo> workerInfoList = this.membershipManager.getCurrentMembership();
        MembershipManager.WorkerInfo workerInfo = null;
        for (MembershipManager.WorkerInfo entry : workerInfoList) {
            if (assignment.getWorkerId().equals(entry.getWorkerId())) {
                workerInfo = entry;
            }
        }
        if (workerInfo == null) {
            InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
            functionStatusBuilder.setRunning(false);
            functionStatusBuilder.setInstanceId(String.valueOf(instanceId));
            functionStatusBuilder.setFailureException("Function has not been scheduled");
            return functionStatusBuilder.build();
        }
        Client client = ClientBuilder.newClient();
        // TODO: implement authentication/authorization
        String jsonResponse = client.target(String.format("http://%s:%d/admin/functions/%s/%s/%s/%d/status", workerInfo.getWorkerHostname(), workerInfo.getPort(), tenant, namespace, functionName, instanceId)).request(MediaType.TEXT_PLAIN).get(String.class);
        InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder();
        try {
            org.apache.pulsar.functions.utils.Utils.mergeJson(jsonResponse, functionStatusBuilder);
        } catch (IOException e) {
            log.warn("Got invalid function status response from {}", workerInfo, e);
            throw new RuntimeException(e);
        }
        functionStatus = functionStatusBuilder.build();
    }
    return functionStatus;
}

Example 4 with RuntimeSpawner

use of org.apache.pulsar.functions.runtime.RuntimeSpawner in project incubator-pulsar by apache.

the class FunctionStatsGeneratorTest method testFunctionsStatsGenerate.

@Test
public void testFunctionsStatsGenerate() {
    FunctionRuntimeManager functionRuntimeManager = mock(FunctionRuntimeManager.class);
    Map<String, FunctionRuntimeInfo> functionRuntimeInfoMap = new HashMap<>();
    WorkerService workerService = mock(WorkerService.class);
    doReturn(functionRuntimeManager).when(workerService).getFunctionRuntimeManager();
    CompletableFuture<InstanceCommunication.MetricsData> metricsDataCompletableFuture = new CompletableFuture<>();
    InstanceCommunication.MetricsData metricsData = InstanceCommunication.MetricsData.newBuilder().putMetrics("__total_processed__", InstanceCommunication.MetricsData.DataDigest.newBuilder().setCount(100.0).setMax(200.0).setSum(300.0).setMin(0.0).build()).putMetrics("__avg_latency_ms__", InstanceCommunication.MetricsData.DataDigest.newBuilder().setCount(10.0).setMax(20.0).setSum(30.0).setMin(0.0).build()).build();
    metricsDataCompletableFuture.complete(metricsData);
    Runtime runtime = mock(Runtime.class);
    doReturn(metricsDataCompletableFuture).when(runtime).getAndResetMetrics();
    RuntimeSpawner runtimeSpawner = mock(RuntimeSpawner.class);
    doReturn(runtime).when(runtimeSpawner).getRuntime();
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build();
    Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build();
    FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class);
    doReturn(runtimeSpawner).when(functionRuntimeInfo).getRuntimeSpawner();
    doReturn(instance).when(functionRuntimeInfo).getFunctionInstance();
    functionRuntimeInfoMap.put(Utils.getFullyQualifiedInstanceId(instance), functionRuntimeInfo);
    doReturn(functionRuntimeInfoMap).when(functionRuntimeManager).getFunctionRuntimeInfos();
    ByteBuf buf = ByteBufAllocator.DEFAULT.heapBuffer();
    SimpleTextOutputStream statsOut = new SimpleTextOutputStream(buf);
    FunctionsStatsGenerator.generate(workerService, "default", statsOut);
    String str = buf.toString(Charset.defaultCharset());
    buf.release();
    Map<String, Metric> metrics = parseMetrics(str);
    Assert.assertEquals(metrics.size(), 8);
    Metric m = metrics.get("pulsar_function__total_processed__count");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 100.0);
    m = metrics.get("pulsar_function__total_processed__max");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 200.0);
    m = metrics.get("pulsar_function__total_processed__sum");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 300.0);
    m = metrics.get("pulsar_function__total_processed__min");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function__avg_latency_ms__count");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 10.0);
    m = metrics.get("pulsar_function__avg_latency_ms__max");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 20.0);
    m = metrics.get("pulsar_function__avg_latency_ms__sum");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 30.0);
    m = metrics.get("pulsar_function__avg_latency_ms__min");
    assertEquals(m.tags.get("cluster"), "default");
    assertEquals(m.tags.get("instanceId"), "0");
    assertEquals(m.tags.get("name"), "func-1");
    assertEquals(m.tags.get("namespace"), "test-tenant/test-namespace");
    assertEquals(m.value, 0.0);
}