Examples with MessageId org.apache.pulsar.client.api.MessageId used on opensource projects

Example 36 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testEffectivelyOnceProcessingFailures.

@Test
public void testEffectivelyOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        // once we get consumer id, simulate receiving 2 messages from consumer
        Message[] msgs = new Message[2];
        for (int i = 1; i <= 2; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            msgs[i - 1] = msg;
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        ProducerInstance producerInstance;
        while (mockProducers.isEmpty()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        producerInstance = mockProducers.values().iterator().next();
        // only first message is published, the second message is not
        Message msg = producerInstance.msgQueue.take();
        assertEquals("message-1!", new String(msg.getData(), UTF_8));
        assertEquals(Utils.getSequenceId(new MessageIdImpl(1L, 1, 0)), msg.getSequenceId());
        assertNull(producerInstance.msgQueue.poll());
        // the first result message is sent but the send future is not completed yet
        // so no acknowledge would happen
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(any(Message.class));
        // since the second message failed to process, for correctness, the instance
        // will close the existing consumer and resubscribe
        ConsumerInstance secondInstance = mockConsumers.get(consumerId);
        while (null == secondInstance || secondInstance == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            secondInstance = mockConsumers.get(consumerId);
        }
        Message secondMsg = mock(Message.class);
        when(secondMsg.getData()).thenReturn("message-2".getBytes(UTF_8));
        when(secondMsg.getMessageId()).thenReturn(new MessageIdImpl(1L, 2, 0));
        secondInstance.addMessage(secondMsg);
        secondInstance.getConf().getMessageListener().received(secondInstance.getConsumer(), secondMsg);
        Message secondReceivedMsg = producerInstance.msgQueue.take();
        assertEquals("message-2!", new String(secondReceivedMsg.getData(), UTF_8));
        assertEquals(Utils.getSequenceId(new MessageIdImpl(1L, 2, 0)), secondReceivedMsg.getSequenceId());
        // the first result message is sent
        verify(secondInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(any(Message.class));
        // complete all the publishes
        synchronized (producerInstance) {
            assertEquals(2, producerInstance.sendFutures.size());
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // all 2 messages are sent
        verify(consumerInstance.getConsumer(), times(1)).acknowledgeCumulativeAsync(same(msgs[0]));
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(msgs[1]));
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(secondMsg));
        verify(secondInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(msgs[0]));
        verify(secondInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(same(msgs[1]));
    }
}

Example 37 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testEffectivelyOnceProcessing.

@Test
public void testEffectivelyOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        ProducerInstance producerInstance;
        while (mockProducers.isEmpty()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        producerInstance = mockProducers.values().iterator().next();
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(Utils.getSequenceId(new MessageIdImpl(1L, i, 0)), msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeCumulativeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
    }
}

Example 38 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method setup.

@BeforeMethod
public void setup() throws Exception {
    mockProducers.clear();
    mockConsumers.clear();
    fnConfig = FunctionConfig.newBuilder().setAutoAck(true).setClassName(TestFunction.class.getName()).addInputs("test-src-topic").setName("test-function").setOutput("test-output-topic").setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setTenant("test-tenant").setNamespace("test-namespace").build();
    config = new InstanceConfig();
    config.setFunctionId("test-function-id");
    config.setFunctionVersion("v1");
    config.setInstanceId("test-instance-id");
    config.setMaxBufferedTuples(1000);
    config.setFunctionConfig(fnConfig);
    mockClient = mock(PulsarClientImpl.class);
    // mock FunctionCacheManager
    fnCache = mock(FunctionCacheManager.class);
    doNothing().when(fnCache).registerFunctionInstance(anyString(), anyString(), anyList(), anyList());
    doNothing().when(fnCache).unregisterFunctionInstance(anyString(), anyString());
    ClassLoader clsLoader = JavaInstanceRunnableTest.class.getClassLoader();
    when(fnCache.getClassLoader(anyString())).thenReturn(clsLoader);
    // mock producer & consumer
    when(mockClient.createProducer(anyString(), any(ProducerConfiguration.class))).thenAnswer(invocationOnMock -> {
        String topic = invocationOnMock.getArgumentAt(0, String.class);
        ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class);
        String producerName = conf.getProducerName();
        Pair<String, String> pair = Pair.of(topic, producerName);
        ProducerInstance producerInstance = mockProducers.get(pair);
        if (null == producerInstance) {
            Producer producer = mock(Producer.class);
            LinkedBlockingQueue<Message> msgQueue = new LinkedBlockingQueue<>();
            final ProducerInstance instance = new ProducerInstance(producer, msgQueue);
            producerInstance = instance;
            when(producer.getProducerName()).thenReturn(producerName);
            when(producer.getTopic()).thenReturn(topic);
            when(producer.sendAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("producer send message {}", msg);
                CompletableFuture<MessageId> future = new CompletableFuture<>();
                instance.addSendFuture(future);
                msgQueue.put(msg);
                return future;
            });
            when(producer.closeAsync()).thenReturn(FutureUtils.Void());
            mockProducers.put(pair, producerInstance);
        }
        return producerInstance.getProducer();
    });
    when(mockClient.subscribe(anyString(), anyString(), any(ConsumerConfiguration.class))).thenAnswer(invocationOnMock -> {
        String topic = invocationOnMock.getArgumentAt(0, String.class);
        String subscription = invocationOnMock.getArgumentAt(1, String.class);
        ConsumerConfiguration conf = invocationOnMock.getArgumentAt(2, ConsumerConfiguration.class);
        Pair<String, String> pair = Pair.of(topic, subscription);
        ConsumerInstance consumerInstance = mockConsumers.get(pair);
        if (null == consumerInstance) {
            Consumer consumer = mock(Consumer.class);
            ConsumerInstance instance = new ConsumerInstance(consumer, conf);
            consumerInstance = instance;
            when(consumer.getTopic()).thenReturn(topic);
            when(consumer.getSubscription()).thenReturn(subscription);
            when(consumer.acknowledgeAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("Ack message {} : message id = {}", msg, msg.getMessageId());
                instance.removeMessage(msg.getMessageId());
                return FutureUtils.Void();
            });
            when(consumer.acknowledgeCumulativeAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId());
                instance.removeMessagesBefore(msg.getMessageId());
                return FutureUtils.Void();
            });
            when(consumer.closeAsync()).thenAnswer(invocationOnMock1 -> {
                mockConsumers.remove(pair, instance);
                return FutureUtils.Void();
            });
            doAnswer(invocationOnMock1 -> {
                mockConsumers.remove(pair, instance);
                return null;
            }).when(consumer).close();
            mockConsumers.put(pair, consumerInstance);
        }
        return consumerInstance.getConsumer();
    });
    // 
    // Mock State Store
    // 
    StorageClientBuilder mockBuilder = mock(StorageClientBuilder.class);
    when(mockBuilder.withNamespace(anyString())).thenReturn(mockBuilder);
    when(mockBuilder.withSettings(any(StorageClientSettings.class))).thenReturn(mockBuilder);
    this.mockStorageClient = mock(StorageClient.class);
    when(mockBuilder.build()).thenReturn(mockStorageClient);
    StorageAdminClient adminClient = mock(StorageAdminClient.class);
    when(mockBuilder.buildAdmin()).thenReturn(adminClient);
    PowerMockito.mockStatic(StorageClientBuilder.class);
    PowerMockito.when(StorageClientBuilder.newBuilder()).thenReturn(mockBuilder);
    when(adminClient.getStream(anyString(), anyString())).thenReturn(FutureUtils.value(StreamProperties.newBuilder().build()));
    mockTable = mock(Table.class);
    when(mockStorageClient.openTable(anyString())).thenReturn(FutureUtils.value(mockTable));
    // 
    // Mock Function Stats
    // 
    mockFunctionStats = spy(new FunctionStats());
    PowerMockito.whenNew(FunctionStats.class).withNoArguments().thenReturn(mockFunctionStats);
    // Mock message builder
    PowerMockito.mockStatic(MessageBuilder.class);
    PowerMockito.when(MessageBuilder.create()).thenAnswer(invocationOnMock -> {
        Message msg = mock(Message.class);
        MessageBuilder builder = mock(MessageBuilder.class);
        when(builder.setContent(any(byte[].class))).thenAnswer(invocationOnMock1 -> {
            byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class);
            when(msg.getData()).thenReturn(content);
            return builder;
        });
        when(builder.setSequenceId(anyLong())).thenAnswer(invocationOnMock1 -> {
            long seqId = invocationOnMock1.getArgumentAt(0, long.class);
            when(msg.getSequenceId()).thenReturn(seqId);
            return builder;
        });
        when(builder.setProperty(anyString(), anyString())).thenAnswer(invocationOnMock1 -> {
            String key = invocationOnMock1.getArgumentAt(0, String.class);
            String value = invocationOnMock1.getArgumentAt(1, String.class);
            when(msg.getProperty(eq(key))).thenReturn(value);
            return builder;
        });
        when(builder.build()).thenReturn(msg);
        return builder;
    });
}

Example 39 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtMostOnceProcessing.

@Test
public void testAtMostOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
    }
}

Example 40 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class SchedulerManager method invokeScheduler.

private void invokeScheduler() {
    List<String> currentMembership = this.membershipManager.getCurrentMembership().stream().map(workerInfo -> workerInfo.getWorkerId()).collect(Collectors.toList());
    List<FunctionMetaData> allFunctions = this.functionMetaDataManager.getAllFunctionMetaData();
    Map<String, Function.Instance> allInstances = computeAllInstances(allFunctions);
    Map<String, Map<String, Assignment>> workerIdToAssignments = this.functionRuntimeManager.getCurrentAssignments();
    // delete assignments of functions and instances that don't exist anymore
    Iterator<Map.Entry<String, Map<String, Assignment>>> it = workerIdToAssignments.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<String, Map<String, Assignment>> workerIdToAssignmentEntry = it.next();
        Map<String, Assignment> functionMap = workerIdToAssignmentEntry.getValue();
        // remove instances that don't exist anymore
        functionMap.entrySet().removeIf(entry -> {
            String fullyQualifiedInstanceId = entry.getKey();
            return !allInstances.containsKey(fullyQualifiedInstanceId);
        });
        // update assignment instances in case attributes of a function gets updated
        for (Map.Entry<String, Assignment> entry : functionMap.entrySet()) {
            String fullyQualifiedInstanceId = entry.getKey();
            Assignment assignment = entry.getValue();
            Function.Instance instance = allInstances.get(fullyQualifiedInstanceId);
            if (!assignment.getInstance().equals(instance)) {
                functionMap.put(fullyQualifiedInstanceId, assignment.toBuilder().setInstance(instance).build());
            }
        }
        if (functionMap.isEmpty()) {
            it.remove();
        }
    }
    List<Assignment> currentAssignments = workerIdToAssignments.entrySet().stream().flatMap(stringMapEntry -> stringMapEntry.getValue().values().stream()).collect(Collectors.toList());
    List<Function.Instance> needsAssignment = this.getUnassignedFunctionInstances(workerIdToAssignments, allInstances);
    List<Assignment> assignments = this.scheduler.schedule(needsAssignment, currentAssignments, currentMembership);
    log.debug("New assignments computed: {}", assignments);
    long assignmentVersion = this.functionRuntimeManager.getCurrentAssignmentVersion() + 1;
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.newBuilder().setVersion(assignmentVersion).addAllAssignments(assignments).build();
    CompletableFuture<MessageId> messageIdCompletableFuture = producer.sendAsync(assignmentsUpdate.toByteArray());
    try {
        messageIdCompletableFuture.get();
    } catch (InterruptedException | ExecutionException e) {
        log.error("Failed to send assignment update", e);
        throw new RuntimeException(e);
    }
    // wait for assignment update to go throw the pipeline
    int retries = 0;
    while (this.functionRuntimeManager.getCurrentAssignmentVersion() < assignmentVersion) {
        if (retries >= this.workerConfig.getAssignmentWriteMaxRetries()) {
            log.warn("Max number of retries reached for waiting for assignment to propagate. Will continue now.");
            break;
        }
        log.info("Waiting for assignments to propagate...");
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        retries++;
    }
}