Search in sources :

Example 61 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class TopicTerminationTest method testSimpleTermination.

@Test
public void testSimpleTermination() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    /* MessageId msgId1 = */
    producer.send("test-msg-1".getBytes());
    /* MessageId msgId2 = */
    producer.send("test-msg-2".getBytes());
    MessageId msgId3 = producer.send("test-msg-3".getBytes());
    MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);
    try {
        producer.send("test-msg-4".getBytes());
        fail("Should have thrown exception");
    } catch (PulsarClientException.TopicTerminatedException e) {
    // Expected
    }
}
Also used : PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 62 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class TopicTerminationTest method testSimpleTerminationReaderListener.

@Test(timeOut = 20000)
public void testSimpleTerminationReaderListener() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    CountDownLatch latch = new CountDownLatch(1);
    Reader<byte[]> reader = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.latest).readerListener(new ReaderListener<byte[]>() {

        @Override
        public void received(Reader<byte[]> reader, Message<byte[]> msg) {
        // do nothing
        }

        @Override
        public void reachedEndOfTopic(Reader<byte[]> reader) {
            latch.countDown();
            assertTrue(reader.hasReachedEndOfTopic());
        }
    }).create();
    /* MessageId msgId1 = */
    producer.send("test-msg-1".getBytes());
    /* MessageId msgId2 = */
    producer.send("test-msg-2".getBytes());
    MessageId msgId3 = producer.send("test-msg-3".getBytes());
    Thread.sleep(100);
    assertFalse(reader.hasReachedEndOfTopic());
    MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);
    assertTrue(latch.await(3, TimeUnit.SECONDS));
    assertTrue(reader.hasReachedEndOfTopic());
}
Also used : Message(org.apache.pulsar.client.api.Message) Reader(org.apache.pulsar.client.api.Reader) CountDownLatch(java.util.concurrent.CountDownLatch) ReaderListener(org.apache.pulsar.client.api.ReaderListener) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 63 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class RawBatchConverter method extractIdsAndKeys.

public static List<ImmutablePair<MessageId, String>> extractIdsAndKeys(RawMessage msg) throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    int batchSize = metadata.getNumMessagesInBatch();
    metadata.recycle();
    List<ImmutablePair<MessageId, String>> idsAndKeys = new ArrayList<>();
    for (int i = 0; i < batchSize; i++) {
        SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder();
        ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(payload, singleMessageMetadataBuilder, 0, batchSize);
        MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
        if (!singleMessageMetadataBuilder.getCompactedOut()) {
            idsAndKeys.add(ImmutablePair.of(id, singleMessageMetadataBuilder.getPartitionKey()));
        }
        singleMessageMetadataBuilder.recycle();
        singleMessagePayload.release();
    }
    return idsAndKeys;
}
Also used : SingleMessageMetadata(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata) MessageMetadata(org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) SingleMessageMetadata(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata) ArrayList(java.util.ArrayList) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) ByteBuf(io.netty.buffer.ByteBuf) MessageId(org.apache.pulsar.client.api.MessageId)

Example 64 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class TwoPhaseCompactor method phaseOneLoop.

private void phaseOneLoop(RawReader reader, Optional<MessageId> firstMessageId, MessageId lastMessageId, Map<String, MessageId> latestForKey, CompletableFuture<PhaseOneResult> loopPromise) {
    if (loopPromise.isDone()) {
        return;
    }
    CompletableFuture<RawMessage> future = reader.readNextAsync();
    scheduleTimeout(future);
    future.whenCompleteAsync((m, exception) -> {
        try {
            if (exception != null) {
                loopPromise.completeExceptionally(exception);
                return;
            }
            MessageId id = m.getMessageId();
            if (RawBatchConverter.isBatch(m)) {
                try {
                    RawBatchConverter.extractIdsAndKeys(m).forEach(e -> latestForKey.put(e.getRight(), e.getLeft()));
                } catch (IOException ioe) {
                    log.info("Error decoding batch for message {}. Whole batch will be included in output", id, ioe);
                }
            } else {
                String key = extractKey(m);
                latestForKey.put(key, id);
            }
            if (id.compareTo(lastMessageId) == 0) {
                loopPromise.complete(new PhaseOneResult(firstMessageId.orElse(id), id, latestForKey));
            } else {
                phaseOneLoop(reader, Optional.of(firstMessageId.orElse(id)), lastMessageId, latestForKey, loopPromise);
            }
        } finally {
            m.close();
        }
    }, scheduler);
}
Also used : IOException(java.io.IOException) RawMessage(org.apache.pulsar.client.api.RawMessage) MessageId(org.apache.pulsar.client.api.MessageId)

Example 65 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class TwoPhaseCompactor method phaseTwoSeekThenLoop.

private CompletableFuture<Long> phaseTwoSeekThenLoop(RawReader reader, MessageId from, MessageId to, Map<String, MessageId> latestForKey, BookKeeper bk, LedgerHandle ledger) {
    CompletableFuture<Long> promise = new CompletableFuture<>();
    reader.seekAsync(from).thenCompose((v) -> {
        Semaphore outstanding = new Semaphore(MAX_OUTSTANDING);
        CompletableFuture<Void> loopPromise = new CompletableFuture<Void>();
        phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise);
        return loopPromise;
    }).thenCompose((v) -> closeLedger(ledger)).thenCompose((v) -> reader.acknowledgeCumulativeAsync(to, ImmutableMap.of(COMPACTED_TOPIC_LEDGER_PROPERTY, ledger.getId()))).whenComplete((res, exception) -> {
        if (exception != null) {
            deleteLedger(bk, ledger).whenComplete((res2, exception2) -> {
                if (exception2 != null) {
                    log.warn("Cleanup of ledger {} for failed", ledger, exception2);
                }
                // complete with original exception
                promise.completeExceptionally(exception);
            });
        } else {
            promise.complete(ledger.getId());
        }
    });
    return promise;
}
Also used : RawBatchConverter(org.apache.pulsar.client.impl.RawBatchConverter) RawMessage(org.apache.pulsar.client.api.RawMessage) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RawReader(org.apache.pulsar.client.api.RawReader) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) ByteBuf(io.netty.buffer.ByteBuf) Map(java.util.Map) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) MessageMetadata(org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata) PulsarClient(org.apache.pulsar.client.api.PulsarClient) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) Commands(org.apache.pulsar.common.api.Commands) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) IOException(java.io.IOException) BookKeeper(org.apache.bookkeeper.client.BookKeeper) BKException(org.apache.bookkeeper.client.BKException) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) MessageId(org.apache.pulsar.client.api.MessageId) Optional(java.util.Optional) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) Semaphore(java.util.concurrent.Semaphore)

Aggregations

MessageId (org.apache.pulsar.client.api.MessageId)65 Test (org.testng.annotations.Test)42 CompletableFuture (java.util.concurrent.CompletableFuture)25 Message (org.apache.pulsar.client.api.Message)22 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)16 List (java.util.List)14 TimeUnit (java.util.concurrent.TimeUnit)14 Producer (org.apache.pulsar.client.api.Producer)14 Future (java.util.concurrent.Future)13 Consumer (org.apache.pulsar.client.api.Consumer)13 MessageIdImpl (org.apache.pulsar.client.impl.MessageIdImpl)13 ExecutorService (java.util.concurrent.ExecutorService)11 Logger (org.slf4j.Logger)11 LoggerFactory (org.slf4j.LoggerFactory)11 ByteBuf (io.netty.buffer.ByteBuf)10 HashSet (java.util.HashSet)10 Map (java.util.Map)10 Lists (com.google.common.collect.Lists)8 IOException (java.io.IOException)8 ArrayList (java.util.ArrayList)8