Examples with FunctionStatsImpl org.apache.pulsar.common.policies.data.FunctionStatsImpl used on opensource projects

Example 1 with FunctionStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionStatsImpl in project pulsar by apache.

the class PulsarFunctionE2ETest method testPulsarFunctionStats.

@Test(timeOut = 20000)
public void testPulsarFunctionStats() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarSink-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, "my.*", sinkTopic, subscriptionName);
    admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
    // try to update function to test: update-function functionality
    admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // validate pulsar sink consumer has started on the topic
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
    // validate stats are empty
    FunctionRuntimeManager functionRuntimeManager = functionsWorkerService.getFunctionRuntimeManager();
    FunctionStatsImpl functionStats = functionRuntimeManager.getFunctionStats(tenant, namespacePortion, functionName, null);
    FunctionStatsImpl functionStatsFromAdmin = (FunctionStatsImpl) admin.functions().getFunctionStats(tenant, namespacePortion, functionName);
    assertEquals(functionStats, functionStatsFromAdmin);
    assertEquals(functionStats.getReceivedTotal(), 0);
    assertEquals(functionStats.getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.getUserExceptionsTotal(), 0);
    assertNull(functionStats.avgProcessLatency);
    assertEquals(functionStats.oneMin.getReceivedTotal(), 0);
    assertEquals(functionStats.oneMin.getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.oneMin.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.oneMin.getUserExceptionsTotal(), 0);
    assertNull(functionStats.oneMin.getAvgProcessLatency());
    assertEquals(functionStats.getAvgProcessLatency(), functionStats.oneMin.getAvgProcessLatency());
    assertNull(functionStats.getLastInvocation());
    assertEquals(functionStats.instances.size(), 1);
    assertEquals(functionStats.instances.get(0).getInstanceId(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0);
    assertNull(functionStats.instances.get(0).getMetrics().getAvgProcessLatency());
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0);
    assertNull(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency());
    assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency());
    assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.getAvgProcessLatency());
    // validate prometheus metrics empty
    String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get());
    log.info("prometheus metrics: {}", prometheusMetrics);
    Map<String, PulsarFunctionTestUtils.Metric> metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics);
    PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_received_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_received_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_user_exceptions_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_user_exceptions_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_process_latency_ms");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, Double.NaN);
    m = metrics.get("pulsar_function_process_latency_ms_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, Double.NaN);
    m = metrics.get("pulsar_function_system_exceptions_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_system_exceptions_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_last_invocation");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_processed_successfully_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_processed_successfully_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    // validate function instance stats empty
    FunctionInstanceStatsDataImpl functionInstanceStats = functionRuntimeManager.getFunctionInstanceStats(tenant, namespacePortion, functionName, 0, null);
    FunctionInstanceStatsDataImpl functionInstanceStatsAdmin = (FunctionInstanceStatsDataImpl) admin.functions().getFunctionStats(tenant, namespacePortion, functionName, 0);
    assertEquals(functionInstanceStats, functionInstanceStatsAdmin);
    assertEquals(functionInstanceStats, functionStats.instances.get(0).getMetrics());
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200);
    // get stats after producing
    functionStats = functionRuntimeManager.getFunctionStats(tenant, namespacePortion, functionName, null);
    functionStatsFromAdmin = (FunctionStatsImpl) admin.functions().getFunctionStats(tenant, namespacePortion, functionName);
    assertEquals(functionStats, functionStatsFromAdmin);
    assertEquals(functionStats.getReceivedTotal(), totalMsgs);
    assertEquals(functionStats.getProcessedSuccessfullyTotal(), totalMsgs);
    assertEquals(functionStats.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.getUserExceptionsTotal(), 0);
    assertTrue(functionStats.avgProcessLatency > 0);
    assertEquals(functionStats.oneMin.getReceivedTotal(), totalMsgs);
    assertEquals(functionStats.oneMin.getProcessedSuccessfullyTotal(), totalMsgs);
    assertEquals(functionStats.oneMin.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.oneMin.getUserExceptionsTotal(), 0);
    assertTrue(functionStats.oneMin.getAvgProcessLatency() > 0);
    assertEquals(functionStats.getAvgProcessLatency(), functionStats.oneMin.getAvgProcessLatency());
    assertTrue(functionStats.getLastInvocation() > 0);
    assertEquals(functionStats.instances.size(), 1);
    assertEquals(functionStats.instances.get(0).getInstanceId(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), totalMsgs);
    assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), totalMsgs);
    assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0);
    assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), totalMsgs);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), totalMsgs);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0);
    assertTrue(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency() > 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency());
    assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.getAvgProcessLatency());
    // validate function instance stats
    functionInstanceStats = functionRuntimeManager.getFunctionInstanceStats(tenant, namespacePortion, functionName, 0, null);
    functionInstanceStatsAdmin = (FunctionInstanceStatsDataImpl) admin.functions().getFunctionStats(tenant, namespacePortion, functionName, 0);
    assertEquals(functionInstanceStats, functionInstanceStatsAdmin);
    assertEquals(functionInstanceStats, functionStats.instances.get(0).getMetrics());
    // validate prometheus metrics
    prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get());
    log.info("prometheus metrics: {}", prometheusMetrics);
    metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics);
    m = metrics.get("pulsar_function_received_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, (double) totalMsgs);
    m = metrics.get("pulsar_function_received_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, (double) totalMsgs);
    m = metrics.get("pulsar_function_user_exceptions_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_user_exceptions_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_process_latency_ms");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_function_process_latency_ms_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_function_system_exceptions_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_system_exceptions_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, 0.0);
    m = metrics.get("pulsar_function_last_invocation");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertTrue(m.value > 0.0);
    m = metrics.get("pulsar_function_processed_successfully_total");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, (double) totalMsgs);
    m = metrics.get("pulsar_function_processed_successfully_total_1min");
    assertEquals(m.tags.get("cluster"), config.getClusterName());
    assertEquals(m.tags.get("instance_id"), "0");
    assertEquals(m.tags.get("name"), functionName);
    assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion));
    assertEquals(m.tags.get("fqfn"), FunctionCommon.getFullyQualifiedName(tenant, namespacePortion, functionName));
    assertEquals(m.value, (double) totalMsgs);
    // delete functions
    admin.functions().deleteFunction(tenant, namespacePortion, functionName);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // make sure subscriptions are cleanup
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 0);
    tempDirectory.assertThatFunctionDownloadTempFilesHaveBeenDeleted();
}

Example 2 with FunctionStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionStatsImpl in project pulsar by apache.

the class FunctionRuntimeManager method getFunctionStats.

/**
 * Get stats of all function instances.
 *
 * @param tenant       the tenant the function belongs to
 * @param namespace    the namespace the function belongs to
 * @param functionName the function name
 * @return a list of function statuses
 * @throws PulsarAdminException
 */
public FunctionStatsImpl getFunctionStats(String tenant, String namespace, String functionName, URI uri) throws PulsarAdminException {
    Collection<Assignment> assignments = this.findFunctionAssignments(tenant, namespace, functionName);
    FunctionStatsImpl functionStats = new FunctionStatsImpl();
    if (assignments.isEmpty()) {
        return functionStats;
    }
    if (runtimeFactory.externallyManaged()) {
        Assignment assignment = assignments.iterator().next();
        boolean isOwner = this.workerConfig.getWorkerId().equals(assignment.getWorkerId());
        if (isOwner) {
            int parallelism = assignment.getInstance().getFunctionMetaData().getFunctionDetails().getParallelism();
            for (int i = 0; i < parallelism; ++i) {
                FunctionInstanceStatsDataImpl functionInstanceStatsData = getFunctionInstanceStats(tenant, namespace, functionName, i, null);
                FunctionInstanceStatsImpl functionInstanceStats = new FunctionInstanceStatsImpl();
                functionInstanceStats.setInstanceId(i);
                functionInstanceStats.setMetrics(functionInstanceStatsData);
                functionStats.addInstance(functionInstanceStats);
            }
        } else {
            // find the hostname/port of the worker who is the owner
            List<WorkerInfo> workerInfoList = this.membershipManager.getCurrentMembership();
            WorkerInfo workerInfo = null;
            for (WorkerInfo entry : workerInfoList) {
                if (assignment.getWorkerId().equals(entry.getWorkerId())) {
                    workerInfo = entry;
                }
            }
            if (workerInfo == null) {
                return functionStats;
            }
            if (uri == null) {
                throw new WebApplicationException(Response.serverError().status(Status.INTERNAL_SERVER_ERROR).build());
            } else {
                URI redirect = UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).build();
                throw new WebApplicationException(Response.temporaryRedirect(redirect).build());
            }
        }
    } else {
        for (Assignment assignment : assignments) {
            boolean isOwner = this.workerConfig.getWorkerId().equals(assignment.getWorkerId());
            FunctionInstanceStatsDataImpl functionInstanceStatsData;
            if (isOwner) {
                functionInstanceStatsData = getFunctionInstanceStats(tenant, namespace, functionName, assignment.getInstance().getInstanceId(), null);
            } else {
                functionInstanceStatsData = (FunctionInstanceStatsDataImpl) this.functionAdmin.functions().getFunctionStats(assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(), assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(), assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(), assignment.getInstance().getInstanceId());
            }
            FunctionInstanceStatsImpl functionInstanceStats = new FunctionInstanceStatsImpl();
            functionInstanceStats.setInstanceId(assignment.getInstance().getInstanceId());
            functionInstanceStats.setMetrics(functionInstanceStatsData);
            functionStats.addInstance(functionInstanceStats);
        }
    }
    return functionStats.calculateOverall();
}

Example 3 with FunctionStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionStatsImpl in project pulsar by apache.

the class PulsarFunctionsTest method testFunctionNegAck.

protected void testFunctionNegAck(Runtime runtime) throws Exception {
    if (functionRuntimeType == FunctionRuntimeType.THREAD) {
        return;
    }
    Schema<?> schema;
    if (Runtime.JAVA == runtime) {
        schema = Schema.STRING;
    } else {
        schema = Schema.BYTES;
    }
    String inputTopicName = "persistent://public/default/test-neg-ack-" + runtime + "-input-" + randomName(8);
    String outputTopicName = "test-neg-ack-" + runtime + "-output-" + randomName(8);
    try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build()) {
        admin.topics().createNonPartitionedTopic(inputTopicName);
        admin.topics().createNonPartitionedTopic(outputTopicName);
    }
    String functionName = "test-neg-ack-fn-" + randomName(8);
    final int numMessages = 20;
    if (runtime == Runtime.PYTHON) {
        submitFunction(runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema);
    } else {
        submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema);
    }
    // get function info
    getFunctionInfoSuccess(functionName);
    // get function stats
    getFunctionStatsEmpty(functionName);
    // publish and consume result
    if (Runtime.JAVA == runtime) {
        // java supports schema
        @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build();
        @Cleanup Consumer<String> consumer = client.newConsumer(Schema.STRING).topic(outputTopicName).subscriptionType(SubscriptionType.Exclusive).subscriptionName("test-sub").subscribe();
        @Cleanup Producer<String> producer = client.newProducer(Schema.STRING).topic(inputTopicName).create();
        for (int i = 0; i < numMessages; i++) {
            producer.send("message-" + i);
        }
        Set<String> expectedMessages = new HashSet<>();
        for (int i = 0; i < numMessages; i++) {
            expectedMessages.add("message-" + i + "!");
        }
        for (int i = 0; i < numMessages; i++) {
            Message<String> msg = consumer.receive(60 * 2, TimeUnit.SECONDS);
            log.info("Received: {}", msg.getValue());
            assertTrue(expectedMessages.contains(msg.getValue()));
            expectedMessages.remove(msg.getValue());
        }
        assertEquals(expectedMessages.size(), 0);
    } else {
        // python doesn't support schema
        @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build();
        @Cleanup Consumer<byte[]> consumer = client.newConsumer(Schema.BYTES).topic(outputTopicName).subscriptionType(SubscriptionType.Exclusive).subscriptionName("test-sub").subscribe();
        @Cleanup Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(inputTopicName).create();
        for (int i = 0; i < numMessages; i++) {
            producer.newMessage().value(("message-" + i).getBytes(UTF_8)).send();
        }
        Set<String> expectedMessages = new HashSet<>();
        for (int i = 0; i < numMessages; i++) {
            expectedMessages.add("message-" + i + "!");
        }
        for (int i = 0; i < numMessages; i++) {
            Message<byte[]> msg = consumer.receive(60 * 2, TimeUnit.SECONDS);
            String msgValue = new String(msg.getValue(), UTF_8);
            log.info("Received: {}", msgValue);
            assertTrue(expectedMessages.contains(msgValue));
            expectedMessages.remove(msgValue);
        }
        assertEquals(expectedMessages.size(), 0);
    }
    // get function status
    ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "status", "--tenant", "public", "--namespace", "default", "--name", functionName);
    FunctionStatus functionStatus = FunctionStatusUtil.decode(result.getStdout());
    assertEquals(functionStatus.getNumInstances(), 1);
    assertEquals(functionStatus.getNumRunning(), 1);
    assertEquals(functionStatus.getInstances().size(), 1);
    assertEquals(functionStatus.getInstances().get(0).getInstanceId(), 0);
    assertTrue(functionStatus.getInstances().get(0).getStatus().getAverageLatency() > 0.0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().isRunning(), true);
    assertTrue(functionStatus.getInstances().get(0).getStatus().getLastInvocationTime() > 0);
    // going to receive two more tuples because of delivery
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumReceived(), numMessages + 2);
    // only going to successfully process 20
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumSuccessfullyProcessed(), numMessages);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumRestarts(), 0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getLatestUserExceptions().size(), 2);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0);
    // get function stats
    result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "stats", "--tenant", "public", "--namespace", "default", "--name", functionName);
    log.info("FUNCTION STATS: {}", result.getStdout());
    FunctionStatsImpl functionStats = FunctionStatsImpl.decode(result.getStdout());
    assertEquals(functionStats.getReceivedTotal(), numMessages + 2);
    assertEquals(functionStats.getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.getUserExceptionsTotal(), 2);
    assertTrue(functionStats.avgProcessLatency > 0);
    assertTrue(functionStats.getLastInvocation() > 0);
    assertEquals(functionStats.instances.size(), 1);
    assertEquals(functionStats.instances.get(0).getInstanceId(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), numMessages + 2);
    assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 2);
    assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0);
    // delete function
    deleteFunction(functionName);
    // get function info
    getFunctionInfoNotFound(functionName);
    // make sure subscriptions are cleanup
    checkSubscriptionsCleanup(inputTopicName);
}

Example 4 with FunctionStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionStatsImpl in project pulsar by apache.

the class PulsarFunctionsTest method getFunctionStatsEmpty.

protected void getFunctionStatsEmpty(String functionName) throws Exception {
    ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "stats", "--tenant", "public", "--namespace", "default", "--name", functionName);
    log.info("FUNCTION STATS: {}", result.getStdout());
    FunctionStatsImpl functionStats = FunctionStatsImpl.decode(result.getStdout());
    assertEquals(functionStats.getReceivedTotal(), 0);
    assertEquals(functionStats.getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.getUserExceptionsTotal(), 0);
    assertEquals(functionStats.avgProcessLatency, null);
    assertEquals(functionStats.oneMin.getReceivedTotal(), 0);
    assertEquals(functionStats.oneMin.getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.oneMin.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.oneMin.getUserExceptionsTotal(), 0);
    assertEquals(functionStats.oneMin.getAvgProcessLatency(), null);
    assertEquals(functionStats.getAvgProcessLatency(), functionStats.oneMin.getAvgProcessLatency());
    assertEquals(functionStats.getLastInvocation(), null);
    assertEquals(functionStats.instances.size(), 1);
    assertEquals(functionStats.instances.get(0).getInstanceId(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), null);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency(), null);
}

Example 5 with FunctionStatsImpl

use of org.apache.pulsar.common.policies.data.FunctionStatsImpl in project pulsar by apache.

the class PulsarFunctionsTest method getFunctionStats.

private void getFunctionStats(String functionName, int numMessages) throws Exception {
    ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "stats", "--tenant", "public", "--namespace", "default", "--name", functionName);
    log.info("FUNCTION STATS: {}", result.getStdout());
    FunctionStatsImpl functionStats = FunctionStatsImpl.decode(result.getStdout());
    assertEquals(functionStats.getReceivedTotal(), numMessages);
    assertEquals(functionStats.getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.getUserExceptionsTotal(), 0);
    assertTrue(functionStats.avgProcessLatency > 0);
    assertEquals(functionStats.oneMin.getReceivedTotal(), numMessages);
    assertEquals(functionStats.oneMin.getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.oneMin.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.oneMin.getUserExceptionsTotal(), 0);
    assertTrue(functionStats.oneMin.getAvgProcessLatency() > 0);
    assertEquals(functionStats.getAvgProcessLatency(), functionStats.oneMin.getAvgProcessLatency());
    assertTrue(functionStats.getLastInvocation() > 0);
    assertEquals(functionStats.instances.size(), 1);
    assertEquals(functionStats.instances.get(0).getInstanceId(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), numMessages);
    assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0);
    assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), numMessages);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0);
    assertTrue(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency() > 0);
}