Examples with FunctionConfig org.apache.pulsar.functions.proto.Function.FunctionConfig used on opensource projects

Example 6 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtLeastOnceProcessingFailures.

@Test
public void testAtLeastOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            if (i % 2 == 0) {
                // all messages (i % 2 == 0) will fail to process.
                continue;
            }
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // only 5 succeed messages are acknowledged
        verify(consumerInstance.getConsumer(), times(5)).acknowledgeAsync(any(Message.class));
        assertEquals(5, consumerInstance.getNumMessages());
        for (int i = 0; i < 10; i++) {
            assertEquals(i % 2 == 0, consumerInstance.containMessage(new MessageIdImpl(1L, i, 0)));
        }
    }
}

Example 7 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtLeastOnceProcessing.

@Test
public void testAtLeastOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
    }
}

Example 8 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtMostOnceProcessingFailures.

@Test
public void testAtMostOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            if (i % 2 == 0) {
                // all messages (i % 2 == 0) will fail to process.
                continue;
            }
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
    }
}

Example 9 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testEffectivelyOnceProcessingFailures.

@Test
public void testEffectivelyOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        // once we get consumer id, simulate receiving 2 messages from consumer
        Message[] msgs = new Message[2];
        for (int i = 1; i <= 2; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            msgs[i - 1] = msg;
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        ProducerInstance producerInstance;
        while (mockProducers.isEmpty()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        producerInstance = mockProducers.values().iterator().next();
        // only first message is published, the second message is not
        Message msg = producerInstance.msgQueue.take();
        assertEquals("message-1!", new String(msg.getData(), UTF_8));
        assertEquals(Utils.getSequenceId(new MessageIdImpl(1L, 1, 0)), msg.getSequenceId());
        assertNull(producerInstance.msgQueue.poll());
        // the first result message is sent but the send future is not completed yet
        // so no acknowledge would happen
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(any(Message.class));
        // since the second message failed to process, for correctness, the instance
        // will close the existing consumer and resubscribe
        ConsumerInstance secondInstance = mockConsumers.get(consumerId);
        while (null == secondInstance || secondInstance == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            secondInstance = mockConsumers.get(consumerId);
        }
        Message secondMsg = mock(Message.class);
        when(secondMsg.getData()).thenReturn("message-2".getBytes(UTF_8));
        when(secondMsg.getMessageId()).thenReturn(new MessageIdImpl(1L, 2, 0));
        secondInstance.addMessage(secondMsg);
        secondInstance.getConf().getMessageListener().received(secondInstance.getConsumer(), secondMsg);
        Message secondReceivedMsg = producerInstance.msgQueue.take();
        assertEquals("message-2!", new String(secondReceivedMsg.getData(), UTF_8));
        assertEquals(Utils.getSequenceId(new MessageIdImpl(1L, 2, 0)), secondReceivedMsg.getSequenceId());
        // the first result message is sent
        verify(secondInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(any(Message.class));
        // complete all the publishes
        synchronized (producerInstance) {
            assertEquals(2, producerInstance.sendFutures.size());
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // all 2 messages are sent
        verify(consumerInstance.getConsumer(), times(1)).acknowledgeCumulativeAsync(same(msgs[0]));
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(msgs[1]));
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(secondMsg));
        verify(secondInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(msgs[0]));
        verify(secondInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(msgs[1]));
    }
}

Example 10 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testVoidFunction.

@Test
public void testVoidFunction() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setClassName(TestVoidFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            String msg = voidFunctionQueue.take();
            log.info("Processed message {}", msg);
            assertEquals("message-" + i, msg);
        }
        // no producer should be initialized
        assertTrue(mockProducers.isEmpty());
    }
}