Examples with MessageIdImpl org.apache.pulsar.client.impl.MessageIdImpl used on opensource projects

Example 21 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtMostOnceProcessing.

@Test
public void testAtMostOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
    }
}

Example 22 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class UtilsTest method testGetSequenceId.

@Test
public void testGetSequenceId() {
    long lid = 12345L;
    long eid = 34566L;
    MessageIdImpl id = mock(MessageIdImpl.class);
    when(id.getLedgerId()).thenReturn(lid);
    when(id.getEntryId()).thenReturn(eid);
    assertEquals((lid << 28) | eid, Utils.getSequenceId(id));
}

Example 23 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class UtilsTest method testGetMessageId.

@Test
public void testGetMessageId() {
    long lid = 12345L;
    long eid = 34566L;
    long sequenceId = (lid << 28) | eid;
    MessageIdImpl id = (MessageIdImpl) Utils.getMessageId(sequenceId);
    assertEquals(lid, id.getLedgerId());
    assertEquals(eid, id.getEntryId());
}

Example 24 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class PulsarKafkaConsumer method poll.

@SuppressWarnings("unchecked")
@Override
public ConsumerRecords<K, V> poll(long timeoutMillis) {
    try {
        QueueItem item = receivedMessages.poll(timeoutMillis, TimeUnit.MILLISECONDS);
        if (item == null) {
            return (ConsumerRecords<K, V>) ConsumerRecords.EMPTY;
        }
        Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();
        int numberOfRecords = 0;
        while (item != null && ++numberOfRecords < MAX_RECORDS_IN_SINGLE_POLL) {
            TopicName topicName = TopicName.get(item.consumer.getTopic());
            String topic = topicName.getPartitionedTopicName();
            int partition = topicName.isPartitioned() ? topicName.getPartitionIndex() : 0;
            Message<byte[]> msg = item.message;
            MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
            long offset = MessageIdUtils.getOffset(msgId);
            TopicPartition tp = new TopicPartition(topic, partition);
            K key = getKey(topic, msg);
            V value = valueDeserializer.deserialize(topic, msg.getData());
            TimestampType timestampType = TimestampType.LOG_APPEND_TIME;
            long timestamp = msg.getPublishTime();
            if (msg.getEventTime() > 0) {
                // If we have Event time, use that in preference
                timestamp = msg.getEventTime();
                timestampType = TimestampType.CREATE_TIME;
            }
            ConsumerRecord<K, V> consumerRecord = new ConsumerRecord<>(topic, partition, offset, timestamp, timestampType, -1, msg.hasKey() ? msg.getKey().length() : 0, msg.getData().length, key, value);
            records.computeIfAbsent(tp, k -> new ArrayList<>()).add(consumerRecord);
            // Update last offset seen by application
            lastReceivedOffset.put(tp, offset);
            // Check if we have an item already available
            item = receivedMessages.poll(0, TimeUnit.MILLISECONDS);
        }
        if (isAutoCommit) {
            // Commit the offset of previously dequeued messages
            commitAsync();
        }
        return new ConsumerRecords<>(records);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}

Example 25 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class MessageIdUtils method getOffset.

public static final long getOffset(MessageId messageId) {
    MessageIdImpl msgId = (MessageIdImpl) messageId;
    long ledgerId = msgId.getLedgerId();
    long entryId = msgId.getEntryId();
    // Combine ledger id and entry id to form offset
    // Use less than 32 bits to represent entry id since it will get
    // rolled over way before overflowing the max int range
    long offset = (ledgerId << 28) | entryId;
    return offset;
}