Examples with FunctionStatus org.apache.pulsar.common.policies.data.FunctionStatus used on opensource projects

Example 1 with FunctionStatus

use of org.apache.pulsar.common.policies.data.FunctionStatus in project pulsar by apache.

the class PulsarFunctionE2ETest method testPulsarFunctionStatus.

@Test(timeOut = 20000)
public void testPulsarFunctionStatus() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarSink-test";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
    FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, "my.*", sinkTopic, subscriptionName);
    admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
    // try to update function to test: update-function functionality
    admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // validate pulsar sink consumer has started on the topic
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(subscriptionName);
            return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200);
    FunctionStatus functionStatus = admin.functions().getFunctionStatus(tenant, namespacePortion, functionName);
    int numInstances = functionStatus.getNumInstances();
    assertEquals(numInstances, 1);
    FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData status = functionStatus.getInstances().get(0).getStatus();
    double count = status.getNumReceived();
    double success = status.getNumSuccessfullyProcessed();
    String ownerWorkerId = status.getWorkerId();
    assertEquals((int) count, totalMsgs);
    assertEquals((int) success, totalMsgs);
    assertEquals(ownerWorkerId, workerId);
    // delete functions
    admin.functions().deleteFunction(tenant, namespacePortion, functionName);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // make sure subscriptions are cleanup
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 0);
}

Example 2 with FunctionStatus

use of org.apache.pulsar.common.policies.data.FunctionStatus in project pulsar by apache.

the class PulsarFunctionE2ETest method testFunctionAutomaticSubCleanup.

@Test(timeOut = 20000)
public void testFunctionAutomaticSubCleanup() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/my-topic1";
    final String sinkTopic = "persistent://" + replNamespace + "/output";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String functionName = "PulsarFunction-test";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // create a producer that creates a topic at broker
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
    FunctionConfig functionConfig = new FunctionConfig();
    functionConfig.setTenant(tenant);
    functionConfig.setNamespace(namespacePortion);
    functionConfig.setName(functionName);
    functionConfig.setParallelism(1);
    functionConfig.setInputs(Collections.singleton(sourceTopic));
    functionConfig.setClassName("org.apache.pulsar.functions.api.examples.ExclamationFunction");
    functionConfig.setOutput(sinkTopic);
    functionConfig.setCleanupSubscription(false);
    functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
    admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
    retryStrategically((test) -> {
        try {
            FunctionConfig configure = admin.functions().getFunction(tenant, namespacePortion, functionName);
            return configure != null && configure.getCleanupSubscription() != null;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    assertFalse(admin.functions().getFunction(tenant, namespacePortion, functionName).getCleanupSubscription());
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // validate pulsar source consumer has started on the topic
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
    // test update cleanup subscription
    functionConfig.setCleanupSubscription(true);
    admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
    retryStrategically((test) -> {
        try {
            return admin.functions().getFunction(tenant, namespacePortion, functionName).getCleanupSubscription();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    assertTrue(admin.functions().getFunction(tenant, namespacePortion, functionName).getCleanupSubscription());
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        String data = "my-message-" + i;
        producer.newMessage().property(propertyKey, propertyValue).value(data).send();
    }
    retryStrategically((test) -> {
        try {
            SubscriptionStats subStats = admin.topics().getStats(sourceTopic).getSubscriptions().get(InstanceUtils.getDefaultSubscriptionName(tenant, namespacePortion, functionName));
            return subStats.getUnackedMessages() == 0 && subStats.getMsgThroughputOut() == totalMsgs;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200);
    FunctionStatus functionStatus = admin.functions().getFunctionStatus(tenant, namespacePortion, functionName);
    int numInstances = functionStatus.getNumInstances();
    assertEquals(numInstances, 1);
    FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData status = functionStatus.getInstances().get(0).getStatus();
    double count = status.getNumReceived();
    double success = status.getNumSuccessfullyProcessed();
    String ownerWorkerId = status.getWorkerId();
    assertEquals((int) count, totalMsgs);
    assertEquals((int) success, totalMsgs);
    assertEquals(ownerWorkerId, workerId);
    // delete functions
    admin.functions().deleteFunction(tenant, namespacePortion, functionName);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // make sure subscriptions are cleanup
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 0);
    /**
     * test do not cleanup subscription *
     */
    functionConfig.setCleanupSubscription(false);
    admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // validate pulsar source consumer has started on the topic
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
    retryStrategically((test) -> {
        try {
            FunctionConfig result = admin.functions().getFunction(tenant, namespacePortion, functionName);
            return !result.getCleanupSubscription();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    assertFalse(admin.functions().getFunction(tenant, namespacePortion, functionName).getCleanupSubscription());
    // test update another config and making sure that subscription cleanup remains unchanged
    functionConfig.setParallelism(2);
    admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl);
    retryStrategically((test) -> {
        try {
            FunctionConfig result = admin.functions().getFunction(tenant, namespacePortion, functionName);
            return result.getParallelism() == 2 && !result.getCleanupSubscription();
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    assertFalse(admin.functions().getFunction(tenant, namespacePortion, functionName).getCleanupSubscription());
    // delete functions
    admin.functions().deleteFunction(tenant, namespacePortion, functionName);
    retryStrategically((test) -> {
        try {
            return admin.topics().getStats(sourceTopic).getSubscriptions().size() == 1;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);
    // make sure subscriptions are cleanup
    assertEquals(admin.topics().getStats(sourceTopic).getSubscriptions().size(), 1);
}

Example 3 with FunctionStatus

use of org.apache.pulsar.common.policies.data.FunctionStatus in project pulsar by apache.

the class PulsarFunctionsTest method testWindowFunction.

protected void testWindowFunction(String type, String[] expectedResults) throws Exception {
    int NUM_OF_MESSAGES = 100;
    int windowLengthCount = 10;
    int slidingIntervalCount = 5;
    String functionName = "test-" + type + "-window-fn-" + randomName(8);
    String inputTopicName = "test-" + type + "-count-window-" + functionRuntimeType + "-input-" + randomName(8);
    String outputTopicName = "test-" + type + "-count-window-" + functionRuntimeType + "-output-" + randomName(8);
    try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build()) {
        admin.topics().createNonPartitionedTopic(inputTopicName);
        admin.topics().createNonPartitionedTopic(outputTopicName);
    }
    CommandGenerator generator = CommandGenerator.createDefaultGenerator(inputTopicName, "org.apache.pulsar.functions.api.examples.WindowDurationFunction");
    generator.setFunctionName(functionName);
    generator.setSinkTopic(outputTopicName);
    generator.setWindowLengthCount(windowLengthCount);
    if (type.equals("sliding")) {
        generator.setSlidingIntervalCount(slidingIntervalCount);
    }
    String[] commands = { "sh", "-c", generator.generateCreateFunctionCommand() };
    ContainerExecResult containerExecResult = pulsarCluster.getAnyWorker().execCmd(commands);
    assertTrue(containerExecResult.getStdout().contains("Created successfully"));
    // get function info
    getFunctionInfoSuccess(functionName);
    containerExecResult = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "status", "--tenant", "public", "--namespace", "default", "--name", functionName);
    FunctionStatus functionStatus = FunctionStatusUtil.decode(containerExecResult.getStdout());
    assertEquals(functionStatus.getNumInstances(), 1);
    assertEquals(functionStatus.getNumRunning(), 1);
    assertEquals(functionStatus.getInstances().size(), 1);
    assertEquals(functionStatus.getInstances().get(0).getInstanceId(), 0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().isRunning(), true);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumReceived(), 0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumSuccessfullyProcessed(), 0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getLatestUserExceptions().size(), 0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0);
    @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build();
    @Cleanup Reader<byte[]> reader = client.newReader().startMessageId(MessageId.earliest).topic(outputTopicName).create();
    @Cleanup Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(inputTopicName).enableBatching(false).create();
    for (int i = 0; i < NUM_OF_MESSAGES; i++) {
        producer.send(String.format("%d", i).getBytes());
    }
    int i = 0;
    while (true) {
        if (i > expectedResults.length) {
            Assertions.fail("More results than expected");
        }
        Message<byte[]> msg = reader.readNext(30, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        String msgStr = new String(msg.getData());
        log.info("[testWindowFunction] i: {} RECV: {}", i, msgStr);
        String result = msgStr.split(":")[0];
        assertThat(result).contains(expectedResults[i]);
        i++;
    }
    getFunctionStatus(functionName, NUM_OF_MESSAGES, true);
    // in case last commit is not updated
    assertThat(i).isGreaterThanOrEqualTo(expectedResults.length - 1);
    deleteFunction(functionName);
    getFunctionInfoNotFound(functionName);
}

Example 4 with FunctionStatus

use of org.apache.pulsar.common.policies.data.FunctionStatus in project pulsar by apache.

the class PulsarFunctionsTest method testFunctionNegAck.

protected void testFunctionNegAck(Runtime runtime) throws Exception {
    if (functionRuntimeType == FunctionRuntimeType.THREAD) {
        return;
    }
    Schema<?> schema;
    if (Runtime.JAVA == runtime) {
        schema = Schema.STRING;
    } else {
        schema = Schema.BYTES;
    }
    String inputTopicName = "persistent://public/default/test-neg-ack-" + runtime + "-input-" + randomName(8);
    String outputTopicName = "test-neg-ack-" + runtime + "-output-" + randomName(8);
    try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build()) {
        admin.topics().createNonPartitionedTopic(inputTopicName);
        admin.topics().createNonPartitionedTopic(outputTopicName);
    }
    String functionName = "test-neg-ack-fn-" + randomName(8);
    final int numMessages = 20;
    if (runtime == Runtime.PYTHON) {
        submitFunction(runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema);
    } else {
        submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema);
    }
    // get function info
    getFunctionInfoSuccess(functionName);
    // get function stats
    getFunctionStatsEmpty(functionName);
    // publish and consume result
    if (Runtime.JAVA == runtime) {
        // java supports schema
        @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build();
        @Cleanup Consumer<String> consumer = client.newConsumer(Schema.STRING).topic(outputTopicName).subscriptionType(SubscriptionType.Exclusive).subscriptionName("test-sub").subscribe();
        @Cleanup Producer<String> producer = client.newProducer(Schema.STRING).topic(inputTopicName).create();
        for (int i = 0; i < numMessages; i++) {
            producer.send("message-" + i);
        }
        Set<String> expectedMessages = new HashSet<>();
        for (int i = 0; i < numMessages; i++) {
            expectedMessages.add("message-" + i + "!");
        }
        for (int i = 0; i < numMessages; i++) {
            Message<String> msg = consumer.receive(60 * 2, TimeUnit.SECONDS);
            log.info("Received: {}", msg.getValue());
            assertTrue(expectedMessages.contains(msg.getValue()));
            expectedMessages.remove(msg.getValue());
        }
        assertEquals(expectedMessages.size(), 0);
    } else {
        // python doesn't support schema
        @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build();
        @Cleanup Consumer<byte[]> consumer = client.newConsumer(Schema.BYTES).topic(outputTopicName).subscriptionType(SubscriptionType.Exclusive).subscriptionName("test-sub").subscribe();
        @Cleanup Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(inputTopicName).create();
        for (int i = 0; i < numMessages; i++) {
            producer.newMessage().value(("message-" + i).getBytes(UTF_8)).send();
        }
        Set<String> expectedMessages = new HashSet<>();
        for (int i = 0; i < numMessages; i++) {
            expectedMessages.add("message-" + i + "!");
        }
        for (int i = 0; i < numMessages; i++) {
            Message<byte[]> msg = consumer.receive(60 * 2, TimeUnit.SECONDS);
            String msgValue = new String(msg.getValue(), UTF_8);
            log.info("Received: {}", msgValue);
            assertTrue(expectedMessages.contains(msgValue));
            expectedMessages.remove(msgValue);
        }
        assertEquals(expectedMessages.size(), 0);
    }
    // get function status
    ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "status", "--tenant", "public", "--namespace", "default", "--name", functionName);
    FunctionStatus functionStatus = FunctionStatusUtil.decode(result.getStdout());
    assertEquals(functionStatus.getNumInstances(), 1);
    assertEquals(functionStatus.getNumRunning(), 1);
    assertEquals(functionStatus.getInstances().size(), 1);
    assertEquals(functionStatus.getInstances().get(0).getInstanceId(), 0);
    assertTrue(functionStatus.getInstances().get(0).getStatus().getAverageLatency() > 0.0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().isRunning(), true);
    assertTrue(functionStatus.getInstances().get(0).getStatus().getLastInvocationTime() > 0);
    // going to receive two more tuples because of delivery
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumReceived(), numMessages + 2);
    // only going to successfully process 20
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumSuccessfullyProcessed(), numMessages);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getNumRestarts(), 0);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getLatestUserExceptions().size(), 2);
    assertEquals(functionStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0);
    // get function stats
    result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "stats", "--tenant", "public", "--namespace", "default", "--name", functionName);
    log.info("FUNCTION STATS: {}", result.getStdout());
    FunctionStatsImpl functionStats = FunctionStatsImpl.decode(result.getStdout());
    assertEquals(functionStats.getReceivedTotal(), numMessages + 2);
    assertEquals(functionStats.getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.getUserExceptionsTotal(), 2);
    assertTrue(functionStats.avgProcessLatency > 0);
    assertTrue(functionStats.getLastInvocation() > 0);
    assertEquals(functionStats.instances.size(), 1);
    assertEquals(functionStats.instances.get(0).getInstanceId(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), numMessages + 2);
    assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), numMessages);
    assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0);
    assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 2);
    assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0);
    // delete function
    deleteFunction(functionName);
    // get function info
    getFunctionInfoNotFound(functionName);
    // make sure subscriptions are cleanup
    checkSubscriptionsCleanup(inputTopicName);
}

Example 5 with FunctionStatus

use of org.apache.pulsar.common.policies.data.FunctionStatus in project pulsar by apache.

the class PulsarFunctionsTest method doGetFunctionStatus.

private void doGetFunctionStatus(String functionName, int numMessages, boolean checkRestarts, int parallelism) throws Exception {
    ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(PulsarCluster.ADMIN_SCRIPT, "functions", "status", "--tenant", "public", "--namespace", "default", "--name", functionName);
    FunctionStatus functionStatus = FunctionStatusUtil.decode(result.getStdout());
    assertEquals(functionStatus.getNumInstances(), parallelism);
    assertEquals(functionStatus.getNumRunning(), parallelism);
    assertEquals(functionStatus.getInstances().size(), parallelism);
    boolean avgLatencyGreaterThanZero = false;
    int totalMessagesProcessed = 0;
    int totalMessagesSuccessfullyProcessed = 0;
    boolean lastInvocationTimeGreaterThanZero = false;
    for (int i = 0; i < parallelism; ++i) {
        assertEquals(functionStatus.getInstances().get(i).getStatus().isRunning(), true);
        assertTrue(functionStatus.getInstances().get(i).getInstanceId() >= 0);
        assertTrue(functionStatus.getInstances().get(i).getInstanceId() < parallelism);
        avgLatencyGreaterThanZero = avgLatencyGreaterThanZero || functionStatus.getInstances().get(i).getStatus().getAverageLatency() > 0.0;
        lastInvocationTimeGreaterThanZero = lastInvocationTimeGreaterThanZero || functionStatus.getInstances().get(i).getStatus().getLastInvocationTime() > 0;
        totalMessagesProcessed += functionStatus.getInstances().get(i).getStatus().getNumReceived();
        totalMessagesSuccessfullyProcessed += functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed();
        if (checkRestarts) {
            assertEquals(functionStatus.getInstances().get(i).getStatus().getNumRestarts(), 0);
        }
        assertEquals(functionStatus.getInstances().get(i).getStatus().getLatestUserExceptions().size(), 0);
        assertEquals(functionStatus.getInstances().get(i).getStatus().getLatestSystemExceptions().size(), 0);
    }
    if (numMessages > 0) {
        assertTrue(avgLatencyGreaterThanZero);
        assertTrue(lastInvocationTimeGreaterThanZero);
    }
    assertEquals(totalMessagesProcessed, numMessages);
    assertEquals(totalMessagesSuccessfullyProcessed, numMessages);
}