use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.
the class PulsarClientImpl method patternTopicSubscribeAsync.
private <T> CompletableFuture<Consumer<T>> patternTopicSubscribeAsync(ConsumerConfigurationData<T> conf, Schema<T> schema) {
String regex = conf.getTopicsPattern().pattern();
TopicName destination = TopicName.get(regex);
NamespaceName namespaceName = destination.getNamespaceObject();
CompletableFuture<Consumer<T>> consumerSubscribedFuture = new CompletableFuture<>();
lookup.getTopicsUnderNamespace(namespaceName).thenAccept(topics -> {
if (log.isDebugEnabled()) {
log.debug("Get topics under namespace {}, topics.size: {}", namespaceName.toString(), topics.size());
topics.forEach(topicName -> log.debug("Get topics under namespace {}, topic: {}", namespaceName.toString(), topicName));
}
List<String> topicsList = topicsPatternFilter(topics, conf.getTopicsPattern());
conf.getTopicNames().addAll(topicsList);
ConsumerBase<T> consumer = new PatternTopicsConsumerImpl<>(conf.getTopicsPattern(), PulsarClientImpl.this, conf, externalExecutorProvider.getExecutor(), consumerSubscribedFuture, schema);
synchronized (consumers) {
consumers.put(consumer, Boolean.TRUE);
}
}).exceptionally(ex -> {
log.warn("[{}] Failed to get topics under namespace", namespaceName);
consumerSubscribedFuture.completeExceptionally(ex);
return null;
});
return consumerSubscribedFuture;
}
use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.
the class TopicsConsumerImpl method subscribeAsync.
// subscribe one more given topic
public CompletableFuture<Void> subscribeAsync(String topicName) {
if (!topicNameValid(topicName)) {
return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Topic name not valid"));
}
if (getState() == State.Closing || getState() == State.Closed) {
return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Topics Consumer was already closed"));
}
CompletableFuture<Void> subscribeResult = new CompletableFuture<>();
final AtomicInteger partitionNumber = new AtomicInteger(0);
client.getPartitionedTopicMetadata(topicName).thenAccept(metadata -> {
if (log.isDebugEnabled()) {
log.debug("Received topic {} metadata.partitions: {}", topicName, metadata.partitions);
}
List<CompletableFuture<Consumer<T>>> futureList;
if (metadata.partitions > 1) {
this.topics.putIfAbsent(topicName, metadata.partitions);
numberTopicPartitions.addAndGet(metadata.partitions);
partitionNumber.addAndGet(metadata.partitions);
futureList = IntStream.range(0, partitionNumber.get()).mapToObj(partitionIndex -> {
String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString();
CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
ConsumerImpl<T> newConsumer = new ConsumerImpl<>(client, partitionName, internalConfig, client.externalExecutorProvider().getExecutor(), partitionIndex, subFuture, schema);
consumers.putIfAbsent(newConsumer.getTopic(), newConsumer);
return subFuture;
}).collect(Collectors.toList());
} else {
this.topics.putIfAbsent(topicName, 1);
numberTopicPartitions.incrementAndGet();
partitionNumber.incrementAndGet();
CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
ConsumerImpl<T> newConsumer = new ConsumerImpl<>(client, topicName, internalConfig, client.externalExecutorProvider().getExecutor(), 0, subFuture, schema);
consumers.putIfAbsent(newConsumer.getTopic(), newConsumer);
futureList = Collections.singletonList(subFuture);
}
FutureUtil.waitForAll(futureList).thenAccept(finalFuture -> {
try {
if (numberTopicPartitions.get() > maxReceiverQueueSize) {
setMaxReceiverQueueSize(numberTopicPartitions.get());
}
int numTopics = this.topics.values().stream().mapToInt(Integer::intValue).sum();
checkState(numberTopicPartitions.get() == numTopics, "numberTopicPartitions " + numberTopicPartitions.get() + " not equals expected: " + numTopics);
// We have successfully created new consumers, so we can start receiving messages for them
startReceivingMessages(consumers.values().stream().filter(consumer1 -> {
String consumerTopicName = consumer1.getTopic();
if (TopicName.get(consumerTopicName).getPartitionedTopicName().equals(TopicName.get(topicName).getPartitionedTopicName().toString())) {
return true;
} else {
return false;
}
}).collect(Collectors.toList()));
subscribeResult.complete(null);
log.info("[{}] [{}] Success subscribe new topic {} in topics consumer, numberTopicPartitions {}", topic, subscription, topicName, numberTopicPartitions.get());
if (this.namespaceName == null) {
this.namespaceName = TopicName.get(topicName).getNamespaceObject();
}
return;
} catch (PulsarClientException e) {
handleSubscribeOneTopicError(topicName, e);
subscribeResult.completeExceptionally(e);
}
}).exceptionally(ex -> {
handleSubscribeOneTopicError(topicName, ex);
subscribeResult.completeExceptionally(ex);
return null;
});
}).exceptionally(ex1 -> {
log.warn("[{}] Failed to get partitioned topic metadata: {}", topicName, ex1.getMessage());
subscribeResult.completeExceptionally(ex1);
return null;
});
return subscribeResult;
}
use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.
the class ReplicatorTest method testResetCursorNotFail.
@Test(enabled = true, timeOut = 30000)
public void testResetCursorNotFail() throws Exception {
log.info("--- Starting ReplicatorTest::testResetCursorNotFail ---");
// This test is to verify that reset cursor fails on global topic
SortedSet<String> testDests = new TreeSet<String>();
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 1; i++) {
final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/resetrepltopic-%d", i));
testDests.add(dest.toString());
results.add(executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
MessageProducer producer1 = new MessageProducer(url1, dest);
log.info("--- Starting producer --- " + url1);
MessageConsumer consumer1 = new MessageConsumer(url1, dest);
log.info("--- Starting Consumer --- " + url1);
// Produce from cluster1 and consume from the rest
producer1.produce(2);
consumer1.receive(2);
producer1.close();
consumer1.close();
return null;
}
}));
}
for (Future<Void> result : results) {
try {
result.get();
} catch (Exception e) {
log.error("exception in getting future result ", e);
fail(String.format("replication test failed with %s exception", e.getMessage()));
}
}
admin1.persistentTopics().resetCursor(testDests.first(), "sub-id", System.currentTimeMillis());
}
use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.
the class ReplicatorTest method testReplicatePeekAndSkip.
@Test(timeOut = 30000)
public void testReplicatePeekAndSkip() throws Exception {
SortedSet<String> testDests = new TreeSet<String>();
final TopicName dest = TopicName.get("persistent://pulsar/global/ns/peekAndSeekTopic");
testDests.add(dest.toString());
MessageProducer producer1 = new MessageProducer(url1, dest);
MessageConsumer consumer1 = new MessageConsumer(url3, dest);
// Produce from cluster1 and consume from the rest
producer1.produce(2);
producer1.close();
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(topic.getReplicators().keys().get(0));
replicator.skipMessages(2);
CompletableFuture<Entry> result = replicator.peekNthMessage(1);
Entry entry = result.get(50, TimeUnit.MILLISECONDS);
assertNull(entry);
consumer1.close();
}
use of org.apache.pulsar.common.naming.TopicName in project incubator-pulsar by apache.
the class ReplicatorTest method testConfigChange.
@Test(enabled = true, timeOut = 30000)
public void testConfigChange() throws Exception {
log.info("--- Starting ReplicatorTest::testConfigChange ---");
// This test is to verify that the config change on global namespace is successfully applied in broker during
// runtime.
// Run a set of producer tasks to create the topics
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 10; i++) {
final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/topic-%d", i));
results.add(executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
MessageProducer producer = new MessageProducer(url1, dest);
log.info("--- Starting producer --- " + url1);
MessageConsumer consumer = new MessageConsumer(url1, dest);
log.info("--- Starting Consumer --- " + url1);
producer.produce(2);
consumer.receive(2);
producer.close();
consumer.close();
return null;
}
}));
}
for (Future<Void> result : results) {
try {
result.get();
} catch (Exception e) {
log.error("exception in getting future result ", e);
fail(String.format("replication test failed with %s exception", e.getMessage()));
}
}
Thread.sleep(1000L);
// Make sure that the internal replicators map contains remote cluster info
ConcurrentOpenHashMap<String, PulsarClient> replicationClients1 = ns1.getReplicationClients();
ConcurrentOpenHashMap<String, PulsarClient> replicationClients2 = ns2.getReplicationClients();
ConcurrentOpenHashMap<String, PulsarClient> replicationClients3 = ns3.getReplicationClients();
Assert.assertNotNull(replicationClients1.get("r2"));
Assert.assertNotNull(replicationClients1.get("r3"));
Assert.assertNotNull(replicationClients2.get("r1"));
Assert.assertNotNull(replicationClients2.get("r3"));
Assert.assertNotNull(replicationClients3.get("r1"));
Assert.assertNotNull(replicationClients3.get("r2"));
// Case 1: Update the global namespace replication configuration to only contains the local cluster itself
admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1"));
// Wait for config changes to be updated.
Thread.sleep(1000L);
// Make sure that the internal replicators map still contains remote cluster info
Assert.assertNotNull(replicationClients1.get("r2"));
Assert.assertNotNull(replicationClients1.get("r3"));
Assert.assertNotNull(replicationClients2.get("r1"));
Assert.assertNotNull(replicationClients2.get("r3"));
Assert.assertNotNull(replicationClients3.get("r1"));
Assert.assertNotNull(replicationClients3.get("r2"));
// Case 2: Update the configuration back
admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
// Wait for config changes to be updated.
Thread.sleep(1000L);
// Make sure that the internal replicators map still contains remote cluster info
Assert.assertNotNull(replicationClients1.get("r2"));
Assert.assertNotNull(replicationClients1.get("r3"));
Assert.assertNotNull(replicationClients2.get("r1"));
Assert.assertNotNull(replicationClients2.get("r3"));
Assert.assertNotNull(replicationClients3.get("r1"));
Assert.assertNotNull(replicationClients3.get("r2"));
// Case 3: TODO: Once automatic cleanup is implemented, add tests case to verify auto removal of clusters
}
Aggregations