Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 1 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BrokerService method createPersistentTopic.

/**
 * It creates a topic async and returns CompletableFuture. It also throttles down configured max-concurrent topic
 * loading and puts them into queue once in-process topics are created.
 *
 * @param topic persistent-topic name
 * @return CompletableFuture<Topic>
 * @throws RuntimeException
 */
protected CompletableFuture<Topic> createPersistentTopic(final String topic) throws RuntimeException {
    checkTopicNsOwnership(topic);
    final CompletableFuture<Topic> topicFuture = new CompletableFuture<>();
    if (!pulsar.getConfiguration().isEnablePersistentTopics()) {
        if (log.isDebugEnabled()) {
            log.debug("Broker is unable to load persistent topic {}", topic);
        }
        topicFuture.completeExceptionally(new NotAllowedException("Broker is not unable to load persistent topic"));
        return topicFuture;
    }
    final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get();
    if (topicLoadSemaphore.tryAcquire()) {
        createPersistentTopic(topic, topicFuture);
        topicFuture.handle((persistentTopic, ex) -> {
            // release permit and process pending topic
            topicLoadSemaphore.release();
            createPendingLoadTopic();
            return null;
        });
    } else {
        pendingTopicLoadingQueue.add(new ImmutablePair<String, CompletableFuture<Topic>>(topic, topicFuture));
        if (log.isDebugEnabled()) {
            log.debug("topic-loading for {} added into pending queue", topic);
        }
    }
    return topicFuture;
}

Example 2 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, CompletableFuture<Topic> topicFuture) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    getManagedLedgerConfig(topicName).thenAccept(managedLedgerConfig -> {
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = new PersistentTopic(topic, ledger, BrokerService.this);
                    CompletableFuture<Void> replicationFuture = persistentTopic.checkReplication();
                    replicationFuture.thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        addTopicToStatsMaps(topicName, persistentTopic);
                        topicFuture.complete(persistentTopic);
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed. Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        });
                        return null;
                    });
                } catch (NamingException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                log.warn("Failed to create topic {}", topic, exception);
                pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
                topicFuture.completeExceptionally(new PersistenceException(exception));
            }
        }, null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}

Example 3 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BrokerService method updateTopicMessageDispatchRate.

private void updateTopicMessageDispatchRate() {
    this.pulsar().getExecutor().submit(() -> {
        // update message-rate for each topic
        topics.forEach((name, topicFuture) -> {
            if (topicFuture.isDone()) {
                String topicName = null;
                try {
                    if (topicFuture.get() instanceof PersistentTopic) {
                        PersistentTopic topic = (PersistentTopic) topicFuture.get();
                        topicName = topicFuture.get().getName();
                        // it first checks namespace-policy rate and if not present then applies broker-config
                        topic.getDispatchRateLimiter().updateDispatchRate();
                    }
                } catch (Exception e) {
                    log.warn("[{}] failed to update message-dispatch rate {}", topicName, e.getMessage());
                }
            }
        });
    });
}

Example 4 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PulsarStats method updateStats.

public synchronized void updateStats(ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, Topic>>> topicsMap) {
    StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf);
    try {
        tempMetricsCollection.clear();
        bundleStats.clear();
        brokerOperabilityMetrics.reset();
        // Json begin
        topicStatsStream.startObject();
        topicsMap.forEach((namespaceName, bundles) -> {
            if (bundles.isEmpty()) {
                return;
            }
            try {
                topicStatsStream.startObject(namespaceName);
                nsStats.reset();
                bundles.forEach((bundle, topics) -> {
                    NamespaceBundleStats currentBundleStats = bundleStats.computeIfAbsent(bundle, k -> new NamespaceBundleStats());
                    currentBundleStats.reset();
                    currentBundleStats.topics = topics.size();
                    topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle));
                    tempNonPersistentTopics.clear();
                    // start persistent topic
                    topicStatsStream.startObject("persistent");
                    topics.forEach((name, topic) -> {
                        if (topic instanceof PersistentTopic) {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", name, e.getMessage(), e);
                            }
                            // this task: helps to activate inactive-backlog-cursors which have caught up and
                            // connected, also deactivate active-backlog-cursors which has backlog
                            ((PersistentTopic) topic).getManagedLedger().checkBackloggedCursors();
                        } else if (topic instanceof NonPersistentTopic) {
                            tempNonPersistentTopics.add((NonPersistentTopic) topic);
                        } else {
                            log.warn("Unsupported type of topic {}", topic.getClass().getName());
                        }
                    });
                    // end persistent topics section
                    topicStatsStream.endObject();
                    if (!tempNonPersistentTopics.isEmpty()) {
                        // start non-persistent topic
                        topicStatsStream.startObject("non-persistent");
                        tempNonPersistentTopics.forEach(topic -> {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", topic.getName(), e.getMessage(), e);
                            }
                        });
                        // end non-persistent topics section
                        topicStatsStream.endObject();
                    }
                    // end namespace-bundle section
                    topicStatsStream.endObject();
                });
                topicStatsStream.endObject();
                // Update metricsCollection with namespace stats
                tempMetricsCollection.add(nsStats.add(namespaceName));
            } catch (Exception e) {
                log.error("Failed to generate namespace stats for namespace {}: {}", namespaceName, e.getMessage(), e);
            }
        });
        if (clusterReplicationMetrics.isMetricsEnabled()) {
            clusterReplicationMetrics.get().forEach(clusterMetric -> tempMetricsCollection.add(clusterMetric));
            clusterReplicationMetrics.reset();
        }
        brokerOperabilityMetrics.getMetrics().forEach(brokerOperabilityMetric -> tempMetricsCollection.add(brokerOperabilityMetric));
        // json end
        topicStatsStream.endObject();
    } catch (Exception e) {
        log.error("Unable to update topic stats", e);
    }
    // swap metricsCollection and tempMetricsCollection
    List<Metrics> tempRefMetrics = metricsCollection;
    metricsCollection = tempMetricsCollection;
    tempMetricsCollection = tempRefMetrics;
    bufferLock.writeLock().lock();
    try {
        ByteBuf tmp = topicStatsBuf;
        topicStatsBuf = tempTopicStatsBuf;
        tempTopicStatsBuf = tmp;
        tempTopicStatsBuf.clear();
    } finally {
        bufferLock.writeLock().unlock();
    }
}

Example 5 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalCreateSubscription.

protected void internalCreateSubscription(String subscriptionName, MessageIdImpl messageId, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    log.info("[{}][{}] Creating subscription {} at message id {}", clientAppId(), topicName, subscriptionName, messageId);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    try {
        if (partitionMetadata.partitions > 0) {
            // Create the subscription on each partition
            List<CompletableFuture<Void>> futures = Lists.newArrayList();
            PulsarAdmin admin = pulsar().getAdminClient();
            for (int i = 0; i < partitionMetadata.partitions; i++) {
                futures.add(admin.persistentTopics().createSubscriptionAsync(topicName.getPartition(i).toString(), subscriptionName, messageId));
            }
            FutureUtil.waitForAll(futures).join();
        } else {
            validateAdminOperationOnTopic(authoritative);
            PersistentTopic topic = (PersistentTopic) getOrCreateTopic(topicName);
            if (topic.getSubscriptions().containsKey(subscriptionName)) {
                throw new RestException(Status.CONFLICT, "Subscription already exists for topic");
            }
            PersistentSubscription subscription = (PersistentSubscription) topic.createSubscription(subscriptionName, InitialPosition.Latest).get();
            subscription.resetCursor(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId())).get();
            log.info("[{}][{}] Successfully created subscription {} at message id {}", clientAppId(), topicName, subscriptionName, messageId);
        }
    } catch (Exception e) {
        Throwable t = e.getCause();
        log.warn("[{}] [{}] Failed to create subscription {} at message id {}", clientAppId(), topicName, subscriptionName, messageId, e);
        if (t instanceof SubscriptionInvalidCursorPosition) {
            throw new RestException(Status.PRECONDITION_FAILED, "Unable to find position for position specified: " + t.getMessage());
        } else {
            throw new RestException(e);
        }
    }
}