use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class TopicTerminationTest method testSimpleTermination.
@Test
public void testSimpleTermination() throws Exception {
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
/* MessageId msgId1 = */
producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */
producer.send("test-msg-2".getBytes());
MessageId msgId3 = producer.send("test-msg-3".getBytes());
MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
assertEquals(lastMessageId, msgId3);
try {
producer.send("test-msg-4".getBytes());
fail("Should have thrown exception");
} catch (PulsarClientException.TopicTerminatedException e) {
// Expected
}
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class TopicTerminationTest method testSimpleTerminationReaderListener.
@Test(timeOut = 20000)
public void testSimpleTerminationReaderListener() throws Exception {
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
CountDownLatch latch = new CountDownLatch(1);
Reader<byte[]> reader = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.latest).readerListener(new ReaderListener<byte[]>() {
@Override
public void received(Reader<byte[]> reader, Message<byte[]> msg) {
// do nothing
}
@Override
public void reachedEndOfTopic(Reader<byte[]> reader) {
latch.countDown();
assertTrue(reader.hasReachedEndOfTopic());
}
}).create();
/* MessageId msgId1 = */
producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */
producer.send("test-msg-2".getBytes());
MessageId msgId3 = producer.send("test-msg-3".getBytes());
Thread.sleep(100);
assertFalse(reader.hasReachedEndOfTopic());
MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
assertEquals(lastMessageId, msgId3);
assertTrue(latch.await(3, TimeUnit.SECONDS));
assertTrue(reader.hasReachedEndOfTopic());
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class RawBatchConverter method extractIdsAndKeys.
public static List<ImmutablePair<MessageId, String>> extractIdsAndKeys(RawMessage msg) throws IOException {
checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
ByteBuf payload = msg.getHeadersAndPayload();
MessageMetadata metadata = Commands.parseMessageMetadata(payload);
int batchSize = metadata.getNumMessagesInBatch();
metadata.recycle();
List<ImmutablePair<MessageId, String>> idsAndKeys = new ArrayList<>();
for (int i = 0; i < batchSize; i++) {
SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder();
ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(payload, singleMessageMetadataBuilder, 0, batchSize);
MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
if (!singleMessageMetadataBuilder.getCompactedOut()) {
idsAndKeys.add(ImmutablePair.of(id, singleMessageMetadataBuilder.getPartitionKey()));
}
singleMessageMetadataBuilder.recycle();
singleMessagePayload.release();
}
return idsAndKeys;
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class TwoPhaseCompactor method phaseOneLoop.
private void phaseOneLoop(RawReader reader, Optional<MessageId> firstMessageId, MessageId lastMessageId, Map<String, MessageId> latestForKey, CompletableFuture<PhaseOneResult> loopPromise) {
if (loopPromise.isDone()) {
return;
}
CompletableFuture<RawMessage> future = reader.readNextAsync();
scheduleTimeout(future);
future.whenCompleteAsync((m, exception) -> {
try {
if (exception != null) {
loopPromise.completeExceptionally(exception);
return;
}
MessageId id = m.getMessageId();
if (RawBatchConverter.isBatch(m)) {
try {
RawBatchConverter.extractIdsAndKeys(m).forEach(e -> latestForKey.put(e.getRight(), e.getLeft()));
} catch (IOException ioe) {
log.info("Error decoding batch for message {}. Whole batch will be included in output", id, ioe);
}
} else {
String key = extractKey(m);
latestForKey.put(key, id);
}
if (id.compareTo(lastMessageId) == 0) {
loopPromise.complete(new PhaseOneResult(firstMessageId.orElse(id), id, latestForKey));
} else {
phaseOneLoop(reader, Optional.of(firstMessageId.orElse(id)), lastMessageId, latestForKey, loopPromise);
}
} finally {
m.close();
}
}, scheduler);
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class TwoPhaseCompactor method phaseTwoSeekThenLoop.
private CompletableFuture<Long> phaseTwoSeekThenLoop(RawReader reader, MessageId from, MessageId to, Map<String, MessageId> latestForKey, BookKeeper bk, LedgerHandle ledger) {
CompletableFuture<Long> promise = new CompletableFuture<>();
reader.seekAsync(from).thenCompose((v) -> {
Semaphore outstanding = new Semaphore(MAX_OUTSTANDING);
CompletableFuture<Void> loopPromise = new CompletableFuture<Void>();
phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise);
return loopPromise;
}).thenCompose((v) -> closeLedger(ledger)).thenCompose((v) -> reader.acknowledgeCumulativeAsync(to, ImmutableMap.of(COMPACTED_TOPIC_LEDGER_PROPERTY, ledger.getId()))).whenComplete((res, exception) -> {
if (exception != null) {
deleteLedger(bk, ledger).whenComplete((res2, exception2) -> {
if (exception2 != null) {
log.warn("Cleanup of ledger {} for failed", ledger, exception2);
}
// complete with original exception
promise.completeExceptionally(exception);
});
} else {
promise.complete(ledger.getId());
}
});
return promise;
}
Aggregations