use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopicE2ETest method testMessageReplay.
/**
* Verify:
* 1. Broker should not replay already acknowledged messages
* 2. Dispatcher should not stuck while dispatching new messages due to previous-replay
* of invalid/already-acked messages
*
* @throws Exception
*/
@Test
public void testMessageReplay() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
Message msg;
int totalMessages = 10;
int replayIndex = totalMessages / 2;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Shared);
conf.setReceiverQueueSize(1);
Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
Producer producer = pulsarClient.createProducer(topicName);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
replayMap.setAccessible(true);
TreeSet<PositionImpl> messagesToReplay = Sets.newTreeSet();
assertNotNull(subRef);
// (1) Produce messages
for (int i = 0; i < totalMessages; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
MessageIdImpl firstAckedMsg = null;
// (2) Consume and ack messages except first message
for (int i = 0; i < totalMessages; i++) {
msg = consumer.receive();
consumer.acknowledge(msg);
MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
if (i == 0) {
firstAckedMsg = msgId;
}
if (i < replayIndex) {
// (3) accumulate acked messages for replay
messagesToReplay.add(new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()));
}
}
// (4) redelivery : should redeliver only unacked messages
Thread.sleep(1000);
replayMap.set(dispatcher, messagesToReplay);
// (a) redelivery with all acked-message should clear messageReply bucket
dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
assertEquals(messagesToReplay.size(), 0);
// (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
messagesToReplay.add(new PositionImpl(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId()));
replayMap.set(dispatcher, messagesToReplay);
// send new message
final String testMsg = "testMsg";
producer.send(testMsg.getBytes());
// consumer should be able to receive only new message and not the
dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
msg = consumer.receive(1, TimeUnit.SECONDS);
assertNotNull(msg);
assertEquals(msg.getData(), testMsg.getBytes());
consumer.close();
producer.close();
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentDispatcherFailoverConsumerTest method testAddRemoveConsumer.
@Test
public void testAddRemoveConsumer() throws Exception {
log.info("--- Starting PersistentDispatcherFailoverConsumerTest::testAddConsumer ---");
PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
PersistentSubscription sub = new PersistentSubscription(topic, cursorMock);
int partitionIndex = 0;
PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock, SubType.Failover, partitionIndex, topic);
// 1. Verify no consumers connected
assertFalse(pdfc.isConsumerConnected());
// 2. Add consumer
Consumer consumer1 = new Consumer(sub, SubType.Exclusive, 1, /* consumer id */
0, "Cons1", /* consumer name */
50000, serverCnx, "myrole-1");
pdfc.addConsumer(consumer1);
List<Consumer> consumers = pdfc.getConsumers();
assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
assertEquals(1, consumers.size());
// 3. Add again, duplicate allowed
pdfc.addConsumer(consumer1);
consumers = pdfc.getConsumers();
assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
assertEquals(2, consumers.size());
// 4. Verify active consumer
assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
// 5. Add another consumer which does not change active consumer
Consumer consumer2 = new Consumer(sub, SubType.Exclusive, 2, /* consumer id */
0, "Cons2", /* consumer name */
50000, serverCnx, "myrole-1");
pdfc.addConsumer(consumer2);
consumers = pdfc.getConsumers();
assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
assertEquals(3, consumers.size());
// 6. Add a consumer which changes active consumer
Consumer consumer0 = new Consumer(sub, SubType.Exclusive, 0, /* consumer id */
0, "Cons0", /* consumer name */
50000, serverCnx, "myrole-1");
pdfc.addConsumer(consumer0);
consumers = pdfc.getConsumers();
assertTrue(pdfc.getActiveConsumer().consumerName() == consumer0.consumerName());
assertEquals(4, consumers.size());
// 7. Remove last consumer
pdfc.removeConsumer(consumer2);
consumers = pdfc.getConsumers();
assertTrue(pdfc.getActiveConsumer().consumerName() == consumer0.consumerName());
assertEquals(3, consumers.size());
// 8. Verify if we can unsubscribe when more than one consumer is connected
assertFalse(pdfc.canUnsubscribe(consumer0));
// 9. Remove active consumer
pdfc.removeConsumer(consumer0);
consumers = pdfc.getConsumers();
assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
assertEquals(2, consumers.size());
// 10. Attempt to remove already removed consumer
String cause = "";
try {
pdfc.removeConsumer(consumer0);
} catch (Exception e) {
cause = e.getMessage();
}
assertEquals(cause, "Consumer was not connected");
// 11. Remove active consumer
pdfc.removeConsumer(consumer1);
consumers = pdfc.getConsumers();
assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
assertEquals(1, consumers.size());
// 11. With only one consumer, unsubscribe is allowed
assertTrue(pdfc.canUnsubscribe(consumer1));
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class BatchMessageTest method testSimpleBatchProducerConsumer.
@Test(dataProvider = "codec")
public void testSimpleBatchProducerConsumer(CompressionType compressionType) throws Exception {
int numMsgs = 500;
int numMsgsInBatch = numMsgs / 20;
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer";
final String subscriptionName = "pc-sub-1" + compressionType.toString();
Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
consumer.close();
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setCompressionType(compressionType);
producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(numMsgsInBatch);
producerConf.setBatchingEnabled(true);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("msg-" + i).getBytes();
Message msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
rolloverPerIntervalStats();
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
consumer = pulsarClient.subscribe(topicName, subscriptionName);
Message lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
Message msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
if (i % 2 == 0) {
consumer.acknowledgeCumulative(msg);
} else {
lastunackedMsg = msg;
}
}
if (lastunackedMsg != null) {
consumer.acknowledgeCumulative(lastunackedMsg);
}
Thread.sleep(100);
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
consumer.close();
producer.close();
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class BrokerService method addTopicToStatsMaps.
private void addTopicToStatsMaps(DestinationName topicName, PersistentTopic topic) {
try {
NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getBundle(topicName);
if (namespaceBundle != null) {
synchronized (multiLayerTopicsMap) {
String serviceUnit = namespaceBundle.toString();
//
multiLayerTopicsMap.computeIfAbsent(topicName.getNamespace(), //
k -> new ConcurrentOpenHashMap<>()).computeIfAbsent(serviceUnit, //
k -> new ConcurrentOpenHashMap<>()).put(topicName.toString(), topic);
}
}
invalidateOfflineTopicStatCache(topicName);
} catch (Exception e) {
log.warn("Got exception when retrieving bundle name during create persistent topic", e);
}
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class Namespaces method clearBacklog.
private void clearBacklog(NamespaceName nsName, String bundleRange, String subscription) {
try {
List<PersistentTopic> topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange);
List<CompletableFuture<Void>> futures = Lists.newArrayList();
if (subscription != null) {
if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) {
subscription = PersistentReplicator.getRemoteCluster(subscription);
}
for (PersistentTopic topic : topicList) {
futures.add(topic.clearBacklog(subscription));
}
} else {
for (PersistentTopic topic : topicList) {
futures.add(topic.clearBacklog());
}
}
FutureUtil.waitForAll(futures).get();
} catch (Exception e) {
log.error("[{}] Failed to clear backlog for namespace {}/{}, subscription: {}", clientAppId(), nsName.toString(), bundleRange, subscription, e);
throw new RestException(e);
}
}
Aggregations