Examples with FunctionInstanceStatsImpl org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl used on opensource projects

Example 1 with FunctionInstanceStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl in project pulsar by apache.

the class FunctionRuntimeManager method getFunctionStats.

/**
 * Get stats of all function instances.
 *
 * @param tenant       the tenant the function belongs to
 * @param namespace    the namespace the function belongs to
 * @param functionName the function name
 * @return a list of function statuses
 * @throws PulsarAdminException
 */
public FunctionStatsImpl getFunctionStats(String tenant, String namespace, String functionName, URI uri) throws PulsarAdminException {
    Collection<Assignment> assignments = this.findFunctionAssignments(tenant, namespace, functionName);
    FunctionStatsImpl functionStats = new FunctionStatsImpl();
    if (assignments.isEmpty()) {
        return functionStats;
    }
    if (runtimeFactory.externallyManaged()) {
        Assignment assignment = assignments.iterator().next();
        boolean isOwner = this.workerConfig.getWorkerId().equals(assignment.getWorkerId());
        if (isOwner) {
            int parallelism = assignment.getInstance().getFunctionMetaData().getFunctionDetails().getParallelism();
            for (int i = 0; i < parallelism; ++i) {
                FunctionInstanceStatsDataImpl functionInstanceStatsData = getFunctionInstanceStats(tenant, namespace, functionName, i, null);
                FunctionInstanceStatsImpl functionInstanceStats = new FunctionInstanceStatsImpl();
                functionInstanceStats.setInstanceId(i);
                functionInstanceStats.setMetrics(functionInstanceStatsData);
                functionStats.addInstance(functionInstanceStats);
            }
        } else {
            // find the hostname/port of the worker who is the owner
            List<WorkerInfo> workerInfoList = this.membershipManager.getCurrentMembership();
            WorkerInfo workerInfo = null;
            for (WorkerInfo entry : workerInfoList) {
                if (assignment.getWorkerId().equals(entry.getWorkerId())) {
                    workerInfo = entry;
                }
            }
            if (workerInfo == null) {
                return functionStats;
            }
            if (uri == null) {
                throw new WebApplicationException(Response.serverError().status(Status.INTERNAL_SERVER_ERROR).build());
            } else {
                URI redirect = UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).build();
                throw new WebApplicationException(Response.temporaryRedirect(redirect).build());
            }
        }
    } else {
        for (Assignment assignment : assignments) {
            boolean isOwner = this.workerConfig.getWorkerId().equals(assignment.getWorkerId());
            FunctionInstanceStatsDataImpl functionInstanceStatsData;
            if (isOwner) {
                functionInstanceStatsData = getFunctionInstanceStats(tenant, namespace, functionName, assignment.getInstance().getInstanceId(), null);
            } else {
                functionInstanceStatsData = (FunctionInstanceStatsDataImpl) this.functionAdmin.functions().getFunctionStats(assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(), assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(), assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(), assignment.getInstance().getInstanceId());
            }
            FunctionInstanceStatsImpl functionInstanceStats = new FunctionInstanceStatsImpl();
            functionInstanceStats.setInstanceId(assignment.getInstance().getInstanceId());
            functionInstanceStats.setMetrics(functionInstanceStatsData);
            functionStats.addInstance(functionInstanceStats);
        }
    }
    return functionStats.calculateOverall();
}

Example 2 with FunctionInstanceStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl in project pulsar by apache.

the class FunctionsImplTest method testMetricsEmpty.

@Test
public void testMetricsEmpty() throws PulsarClientException {
    Function.FunctionDetails.Builder functionDetailsBuilder = createDefaultFunctionDetails().toBuilder();
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionDetails(functionDetailsBuilder.build());
    instanceConfig.setMaxBufferedTuples(1024);
    JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable(instanceConfig, null, null, null, null, null, null, null, null);
    CompletableFuture<InstanceCommunication.MetricsData> completableFuture = new CompletableFuture<InstanceCommunication.MetricsData>();
    completableFuture.complete(javaInstanceRunnable.getMetrics());
    Runtime runtime = mock(Runtime.class);
    doReturn(completableFuture).when(runtime).getMetrics(anyInt());
    RuntimeSpawner runtimeSpawner = mock(RuntimeSpawner.class);
    doReturn(runtime).when(runtimeSpawner).getRuntime();
    FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class);
    doReturn(runtimeSpawner).when(functionRuntimeInfo).getRuntimeSpawner();
    FunctionInstanceStatsImpl instanceStats1 = WorkerUtils.getFunctionInstanceStats("public/default/test", functionRuntimeInfo, 0);
    FunctionInstanceStatsImpl instanceStats2 = WorkerUtils.getFunctionInstanceStats("public/default/test", functionRuntimeInfo, 1);
    FunctionStatsImpl functionStats = new FunctionStatsImpl();
    functionStats.addInstance(instanceStats1);
    functionStats.addInstance(instanceStats2);
    assertNotNull(functionStats.calculateOverall());
}

Example 3 with FunctionInstanceStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl in project pulsar by yahoo.

the class WorkerImpl method getFunctionsMetrics.

@Override
public List<WorkerFunctionInstanceStats> getFunctionsMetrics(String clientRole) throws IOException {
    if (!isWorkerServiceAvailable()) {
        throwUnavailableException();
    }
    if (worker().getWorkerConfig().isAuthorizationEnabled() && !isSuperUser(clientRole)) {
        log.error("Client [{}] is not authorized to get function stats", clientRole);
        throw new RestException(Status.UNAUTHORIZED, "Client is not authorized to perform operation");
    }
    Map<String, FunctionRuntimeInfo> functionRuntimes = worker().getFunctionRuntimeManager().getFunctionRuntimeInfos();
    List<WorkerFunctionInstanceStats> metricsList = new ArrayList<>(functionRuntimes.size());
    for (Map.Entry<String, FunctionRuntimeInfo> entry : functionRuntimes.entrySet()) {
        String fullyQualifiedInstanceName = entry.getKey();
        FunctionRuntimeInfo functionRuntimeInfo = entry.getValue();
        if (worker().getFunctionRuntimeManager().getRuntimeFactory().externallyManaged()) {
            Function.FunctionDetails functionDetails = functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionDetails();
            int parallelism = functionDetails.getParallelism();
            for (int i = 0; i < parallelism; ++i) {
                FunctionInstanceStatsImpl functionInstanceStats = WorkerUtils.getFunctionInstanceStats(fullyQualifiedInstanceName, functionRuntimeInfo, i);
                WorkerFunctionInstanceStats workerFunctionInstanceStats = new WorkerFunctionInstanceStats();
                workerFunctionInstanceStats.setName(FunctionCommon.getFullyQualifiedInstanceId(functionDetails.getTenant(), functionDetails.getNamespace(), functionDetails.getName(), i));
                workerFunctionInstanceStats.setMetrics(functionInstanceStats.getMetrics());
                metricsList.add(workerFunctionInstanceStats);
            }
        } else {
            FunctionInstanceStatsImpl functionInstanceStats = WorkerUtils.getFunctionInstanceStats(fullyQualifiedInstanceName, functionRuntimeInfo, functionRuntimeInfo.getFunctionInstance().getInstanceId());
            WorkerFunctionInstanceStats workerFunctionInstanceStats = new WorkerFunctionInstanceStats();
            workerFunctionInstanceStats.setName(fullyQualifiedInstanceName);
            workerFunctionInstanceStats.setMetrics(functionInstanceStats.getMetrics());
            metricsList.add(workerFunctionInstanceStats);
        }
    }
    return metricsList;
}

Example 4 with FunctionInstanceStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl in project incubator-pulsar by apache.

the class WorkerUtils method getFunctionInstanceStats.

public static FunctionInstanceStatsImpl getFunctionInstanceStats(String fullyQualifiedInstanceName, FunctionRuntimeInfo functionRuntimeInfo, int instanceId) {
    RuntimeSpawner functionRuntimeSpawner = functionRuntimeInfo.getRuntimeSpawner();
    FunctionInstanceStatsImpl functionInstanceStats = new FunctionInstanceStatsImpl();
    if (functionRuntimeSpawner != null) {
        Runtime functionRuntime = functionRuntimeSpawner.getRuntime();
        if (functionRuntime != null) {
            try {
                InstanceCommunication.MetricsData metricsData = functionRuntime.getMetrics(instanceId).get();
                functionInstanceStats.setInstanceId(instanceId);
                FunctionInstanceStatsDataImpl functionInstanceStatsData = new FunctionInstanceStatsDataImpl();
                functionInstanceStatsData.setReceivedTotal(metricsData.getReceivedTotal());
                functionInstanceStatsData.setProcessedSuccessfullyTotal(metricsData.getProcessedSuccessfullyTotal());
                functionInstanceStatsData.setSystemExceptionsTotal(metricsData.getSystemExceptionsTotal());
                functionInstanceStatsData.setUserExceptionsTotal(metricsData.getUserExceptionsTotal());
                functionInstanceStatsData.setAvgProcessLatency(metricsData.getAvgProcessLatency() == 0.0 ? null : metricsData.getAvgProcessLatency());
                functionInstanceStatsData.setLastInvocation(metricsData.getLastInvocation() == 0 ? null : metricsData.getLastInvocation());
                functionInstanceStatsData.oneMin.setReceivedTotal(metricsData.getReceivedTotal1Min());
                functionInstanceStatsData.oneMin.setProcessedSuccessfullyTotal(metricsData.getProcessedSuccessfullyTotal1Min());
                functionInstanceStatsData.oneMin.setSystemExceptionsTotal(metricsData.getSystemExceptionsTotal1Min());
                functionInstanceStatsData.oneMin.setUserExceptionsTotal(metricsData.getUserExceptionsTotal1Min());
                functionInstanceStatsData.oneMin.setAvgProcessLatency(metricsData.getAvgProcessLatency1Min() == 0.0 ? null : metricsData.getAvgProcessLatency1Min());
                // Filter out values that are NaN
                Map<String, Double> statsDataMap = metricsData.getUserMetricsMap().entrySet().stream().filter(stringDoubleEntry -> !stringDoubleEntry.getValue().isNaN()).collect(Collectors.toMap(x -> x.getKey(), x -> x.getValue()));
                functionInstanceStatsData.setUserMetrics(statsDataMap);
                functionInstanceStats.setMetrics(functionInstanceStatsData);
            } catch (InterruptedException | ExecutionException e) {
                log.warn("Failed to collect metrics for function instance {}", fullyQualifiedInstanceName, e);
            }
        }
    }
    return functionInstanceStats;
}

Example 5 with FunctionInstanceStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl in project incubator-pulsar by apache.

the class FunctionsImplTest method testMetricsEmpty.

@Test
public void testMetricsEmpty() throws PulsarClientException {
    Function.FunctionDetails.Builder functionDetailsBuilder = createDefaultFunctionDetails().toBuilder();
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionDetails(functionDetailsBuilder.build());
    instanceConfig.setMaxBufferedTuples(1024);
    JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable(instanceConfig, null, null, null, null, null, null, null, null);
    CompletableFuture<InstanceCommunication.MetricsData> completableFuture = new CompletableFuture<InstanceCommunication.MetricsData>();
    completableFuture.complete(javaInstanceRunnable.getMetrics());
    Runtime runtime = mock(Runtime.class);
    doReturn(completableFuture).when(runtime).getMetrics(anyInt());
    RuntimeSpawner runtimeSpawner = mock(RuntimeSpawner.class);
    doReturn(runtime).when(runtimeSpawner).getRuntime();
    FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class);
    doReturn(runtimeSpawner).when(functionRuntimeInfo).getRuntimeSpawner();
    FunctionInstanceStatsImpl instanceStats1 = WorkerUtils.getFunctionInstanceStats("public/default/test", functionRuntimeInfo, 0);
    FunctionInstanceStatsImpl instanceStats2 = WorkerUtils.getFunctionInstanceStats("public/default/test", functionRuntimeInfo, 1);
    FunctionStatsImpl functionStats = new FunctionStatsImpl();
    functionStats.addInstance(instanceStats1);
    functionStats.addInstance(instanceStats2);
    assertNotNull(functionStats.calculateOverall());
}