Examples with SystemTopic org.apache.pulsar.broker.service.persistent.SystemTopic used on opensource projects

Example 1 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project pulsar by apache.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture<Optional<Topic>> topicFuture, Map<String, String> properties) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    if (isTransactionSystemTopic(topicName)) {
        String msg = String.format("Can not create transaction system topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new NotAllowedException(msg));
        return;
    }
    CompletableFuture<Void> maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null);
    maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> {
        if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) {
            // init managedLedger interceptor
            Set<BrokerEntryMetadataInterceptor> interceptors = new HashSet<>();
            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
                // add individual AppendOffsetMetadataInterceptor for each topic
                if (interceptor instanceof AppendIndexMetadataInterceptor) {
                    interceptors.add(new AppendIndexMetadataInterceptor());
                } else {
                    interceptors.add(interceptor);
                }
            }
            managedLedgerConfig.setManagedLedgerInterceptor(new ManagedLedgerInterceptorImpl(interceptors, brokerEntryPayloadProcessors));
        }
        managedLedgerConfig.setCreateIfMissing(createIfMissing);
        managedLedgerConfig.setProperties(properties);
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : new PersistentTopic(topic, ledger, BrokerService.this);
                    CompletableFuture<Void> preCreateSubForCompaction = persistentTopic.preCreateSubscriptionForCompactionIfNeeded();
                    CompletableFuture<Void> replicationFuture = persistentTopic.initialize().thenCompose(__ -> persistentTopic.checkReplication());
                    CompletableFuture.allOf(preCreateSubForCompaction, replicationFuture).thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        if (topicFuture.isCompletedExceptionally()) {
                            log.warn("{} future is already completed with failure {}, closing the topic", topic, FutureUtil.getException(topicFuture));
                            persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                                topics.remove(topic, topicFuture);
                            });
                        } else {
                            addTopicToStatsMaps(topicName, persistentTopic);
                            topicFuture.complete(Optional.of(persistentTopic));
                        }
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        });
                        return null;
                    });
                } catch (PulsarServerException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) {
                    // We were just trying to load a topic and the topic doesn't exist
                    topicFuture.complete(Optional.empty());
                } else {
                    log.warn("Failed to create topic {}", topic, exception);
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(new PersistenceException(exception));
                }
            }
        }, () -> isTopicNsOwnedByBroker(topicName), null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}

Example 2 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project pulsar by yahoo.

the class CompactionTest method testHealthCheckTopicNotCompacted.

@SneakyThrows
@Test
public void testHealthCheckTopicNotCompacted() {
    NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck";
    NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck";
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicV1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicV2).create();
    Optional<Topic> topicReferenceV1 = pulsar.getBrokerService().getTopic(topicV1, false).join();
    Optional<Topic> topicReferenceV2 = pulsar.getBrokerService().getTopic(topicV2, false).join();
    assertFalse(((SystemTopic) topicReferenceV1.get()).isCompactionEnabled());
    assertFalse(((SystemTopic) topicReferenceV2.get()).isCompactionEnabled());
    producer1.close();
    producer2.close();
}

Example 3 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project incubator-pulsar by apache.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture<Optional<Topic>> topicFuture, Map<String, String> properties) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    if (isTransactionInternalName(topicName)) {
        String msg = String.format("Can not create transaction system topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new NotAllowedException(msg));
        return;
    }
    CompletableFuture<Void> maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null);
    maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> {
        if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) {
            // init managedLedger interceptor
            Set<BrokerEntryMetadataInterceptor> interceptors = new HashSet<>();
            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
                // add individual AppendOffsetMetadataInterceptor for each topic
                if (interceptor instanceof AppendIndexMetadataInterceptor) {
                    interceptors.add(new AppendIndexMetadataInterceptor());
                } else {
                    interceptors.add(interceptor);
                }
            }
            managedLedgerConfig.setManagedLedgerInterceptor(new ManagedLedgerInterceptorImpl(interceptors, brokerEntryPayloadProcessors));
        }
        managedLedgerConfig.setCreateIfMissing(createIfMissing);
        managedLedgerConfig.setProperties(properties);
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : new PersistentTopic(topic, ledger, BrokerService.this);
                    persistentTopic.initialize().thenCompose(__ -> persistentTopic.preCreateSubscriptionForCompactionIfNeeded()).thenCompose(__ -> persistentTopic.checkReplication()).thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        if (topicFuture.isCompletedExceptionally()) {
                            log.warn("{} future is already completed with failure {}, closing the" + " topic", topic, FutureUtil.getException(topicFuture));
                            persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                                topics.remove(topic, topicFuture);
                            }, executor());
                        } else {
                            addTopicToStatsMaps(topicName, persistentTopic);
                            topicFuture.complete(Optional.of(persistentTopic));
                        }
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        }, executor());
                        return null;
                    });
                } catch (PulsarServerException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) {
                    // We were just trying to load a topic and the topic doesn't exist
                    topicFuture.complete(Optional.empty());
                } else {
                    log.warn("Failed to create topic {}", topic, exception);
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(new PersistenceException(exception));
                }
            }
        }, () -> isTopicNsOwnedByBroker(topicName), null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}

Example 4 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project incubator-pulsar by apache.

the class CompactionTest method testHealthCheckTopicNotCompacted.

@SneakyThrows
@Test
public void testHealthCheckTopicNotCompacted() {
    NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck";
    NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck";
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicV1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicV2).create();
    Optional<Topic> topicReferenceV1 = pulsar.getBrokerService().getTopic(topicV1, false).join();
    Optional<Topic> topicReferenceV2 = pulsar.getBrokerService().getTopic(topicV2, false).join();
    assertFalse(((SystemTopic) topicReferenceV1.get()).isCompactionEnabled());
    assertFalse(((SystemTopic) topicReferenceV2.get()).isCompactionEnabled());
    producer1.close();
    producer2.close();
}

Example 5 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project pulsar by yahoo.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture<Optional<Topic>> topicFuture, Map<String, String> properties) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    if (isTransactionInternalName(topicName)) {
        String msg = String.format("Can not create transaction system topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new NotAllowedException(msg));
        return;
    }
    CompletableFuture<Void> maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null);
    maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> {
        if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) {
            // init managedLedger interceptor
            Set<BrokerEntryMetadataInterceptor> interceptors = new HashSet<>();
            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
                // add individual AppendOffsetMetadataInterceptor for each topic
                if (interceptor instanceof AppendIndexMetadataInterceptor) {
                    interceptors.add(new AppendIndexMetadataInterceptor());
                } else {
                    interceptors.add(interceptor);
                }
            }
            managedLedgerConfig.setManagedLedgerInterceptor(new ManagedLedgerInterceptorImpl(interceptors, brokerEntryPayloadProcessors));
        }
        managedLedgerConfig.setCreateIfMissing(createIfMissing);
        managedLedgerConfig.setProperties(properties);
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : new PersistentTopic(topic, ledger, BrokerService.this);
                    persistentTopic.initialize().thenCompose(__ -> persistentTopic.preCreateSubscriptionForCompactionIfNeeded()).thenCompose(__ -> persistentTopic.checkReplication()).thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        if (topicFuture.isCompletedExceptionally()) {
                            log.warn("{} future is already completed with failure {}, closing the" + " topic", topic, FutureUtil.getException(topicFuture));
                            persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                                topics.remove(topic, topicFuture);
                            }, executor());
                        } else {
                            addTopicToStatsMaps(topicName, persistentTopic);
                            topicFuture.complete(Optional.of(persistentTopic));
                        }
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        }, executor());
                        return null;
                    });
                } catch (PulsarServerException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) {
                    // We were just trying to load a topic and the topic doesn't exist
                    topicFuture.complete(Optional.empty());
                } else {
                    log.warn("Failed to create topic {}", topic, exception);
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(new PersistenceException(exception));
                }
            }
        }, () -> isTopicNsOwnedByBroker(topicName), null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}