Search in sources :

Example 1 with PersistenceException

use of com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException in project pulsar by yahoo.

the class PersistentTopic method unsubscribe.

/**
     * Delete the cursor ledger for a given subscription
     *
     * @param subscriptionName
     *            Subscription for which the cursor ledger is to be deleted
     * @return Completable future indicating completion of unsubscribe operation Completed exceptionally with:
     *         ManagedLedgerException if cursor ledger delete fails
     */
@Override
public CompletableFuture<Void> unsubscribe(String subscriptionName) {
    CompletableFuture<Void> unsubscribeFuture = new CompletableFuture<>();
    ledger.asyncDeleteCursor(Codec.encode(subscriptionName), new DeleteCursorCallback() {

        @Override
        public void deleteCursorComplete(Object ctx) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{}] Cursor deleted successfully", topic, subscriptionName);
            }
            subscriptions.remove(subscriptionName);
            unsubscribeFuture.complete(null);
            lastActive = System.nanoTime();
        }

        @Override
        public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{}] Error deleting cursor for subscription", topic, subscriptionName, exception);
            }
            unsubscribeFuture.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return unsubscribeFuture;
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback)

Example 2 with PersistenceException

use of com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException in project pulsar by yahoo.

the class PersistentTopic method startReplicator.

CompletableFuture<Void> startReplicator(String remoteCluster) {
    log.info("[{}] Starting replicator to remote: {}", topic, remoteCluster);
    final CompletableFuture<Void> future = new CompletableFuture<>();
    String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster);
    ledger.asyncOpenCursor(name, new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
            replicators.computeIfAbsent(remoteCluster, r -> new PersistentReplicator(PersistentTopic.this, cursor, localCluster, remoteCluster, brokerService));
            future.complete(null);
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            future.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return future;
}
Also used : ReplicationMetrics(com.yahoo.pulsar.broker.stats.ReplicationMetrics) SubType(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType) PersistentSubscriptionStats(com.yahoo.pulsar.common.policies.data.PersistentSubscriptionStats) LedgerInfo(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.LedgerInfo) NamingException(com.yahoo.pulsar.broker.service.BrokerServiceException.NamingException) ConsumerStats(com.yahoo.pulsar.common.policies.data.ConsumerStats) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) PersistentTopicInternalStats(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats) LoggerFactory(org.slf4j.LoggerFactory) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) NamespaceBundleStats(com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats) Policies(com.yahoo.pulsar.common.policies.data.Policies) BacklogQuota(com.yahoo.pulsar.common.policies.data.BacklogQuota) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) TopicBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.TopicBusyException) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ReplicatorStats(com.yahoo.pulsar.common.policies.data.ReplicatorStats) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) Objects(com.google.common.base.Objects) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) CursorStats(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) Instant(java.time.Instant) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) TopicFencedException(com.yahoo.pulsar.broker.service.BrokerServiceException.TopicFencedException) Lists(com.beust.jcommander.internal.Lists) ZoneId(java.time.ZoneId) Sets(com.google.common.collect.Sets) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Topic(com.yahoo.pulsar.broker.service.Topic) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) Consumer(com.yahoo.pulsar.broker.service.Consumer) List(java.util.List) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) StatsOutputStream(com.yahoo.pulsar.utils.StatsOutputStream) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) UnsupportedVersionException(com.yahoo.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) SubscriptionBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) MessageImpl(com.yahoo.pulsar.client.impl.MessageImpl) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) ServerCnx(com.yahoo.pulsar.broker.service.ServerCnx) ByteBuf(io.netty.buffer.ByteBuf) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) ConcurrentOpenHashSet(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet) ConsumerBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) Codec(com.yahoo.pulsar.common.util.Codec) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) ClusterReplicationMetrics(com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) BrokerServiceException(com.yahoo.pulsar.broker.service.BrokerServiceException) PublisherStats(com.yahoo.pulsar.common.policies.data.PublisherStats) Producer(com.yahoo.pulsar.broker.service.Producer) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) AtomicLongFieldUpdater(java.util.concurrent.atomic.AtomicLongFieldUpdater) NamespaceStats(com.yahoo.pulsar.broker.stats.NamespaceStats) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) DateTimeFormatter(java.time.format.DateTimeFormatter) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor)

Example 3 with PersistenceException

use of com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException in project pulsar by yahoo.

the class PersistentSubscription method close.

/**
     * Close the cursor ledger for this subscription. Requires that there are no active consumers on the dispatcher
     *
     * @return CompletableFuture indicating the completion of delete operation
     */
@Override
public CompletableFuture<Void> close() {
    CompletableFuture<Void> closeFuture = new CompletableFuture<>();
    synchronized (this) {
        if (dispatcher != null && dispatcher.isConsumerConnected()) {
            closeFuture.completeExceptionally(new SubscriptionBusyException("Subscription has active consumers"));
            return closeFuture;
        }
        IS_FENCED_UPDATER.set(this, TRUE);
        log.info("[{}][{}] Successfully fenced cursor ledger [{}]", topicName, subName, cursor);
    }
    cursor.asyncClose(new CloseCallback() {

        @Override
        public void closeComplete(Object ctx) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{}] Successfully closed cursor ledger", topicName, subName);
            }
            closeFuture.complete(null);
        }

        @Override
        public void closeFailed(ManagedLedgerException exception, Object ctx) {
            IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE);
            log.error("[{}][{}] Error closing cursor for subscription", topicName, subName, exception);
            closeFuture.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return closeFuture;
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) SubscriptionBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException)

Example 4 with PersistenceException

use of com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException in project pulsar by yahoo.

the class BrokerService method createPersistentTopic.

private CompletableFuture<Topic> createPersistentTopic(final String topic) throws RuntimeException {
    checkTopicNsOwnership(topic);
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    DestinationName destinationName = DestinationName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(destinationName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        throw new RuntimeException(new ServiceUnitNotReadyException(msg));
    }
    final CompletableFuture<Topic> topicFuture = new CompletableFuture<>();
    getManagedLedgerConfig(destinationName).thenAccept(config -> {
        managedLedgerFactory.asyncOpen(destinationName.getPersistenceNamingEncoding(), config, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                PersistentTopic persistentTopic = new PersistentTopic(topic, ledger, BrokerService.this);
                CompletableFuture<Void> replicationFuture = persistentTopic.checkReplication();
                replicationFuture.thenRun(() -> {
                    log.info("Created topic {}", topic);
                    long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                    pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                    addTopicToStatsMaps(destinationName, persistentTopic);
                    topicFuture.complete(persistentTopic);
                });
                replicationFuture.exceptionally((ex) -> {
                    log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex);
                    persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                        topics.remove(topic, topicFuture);
                        topicFuture.completeExceptionally(ex);
                    });
                    return null;
                });
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                log.warn("Failed to create topic {}", topic, exception);
                topics.remove(topic, topicFuture);
                topicFuture.completeExceptionally(new PersistenceException(exception));
            }
        }, null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        topics.remove(topic, topicFuture);
        topicFuture.completeExceptionally(exception);
        return null;
    });
    return topicFuture;
}
Also used : DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) LoggerFactory(org.slf4j.LoggerFactory) NamespaceBundleStats(com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats) Stat(org.apache.zookeeper.data.Stat) Policies(com.yahoo.pulsar.common.policies.data.Policies) EpollMode(io.netty.channel.epoll.EpollMode) PersistencePolicies(com.yahoo.pulsar.common.policies.data.PersistencePolicies) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PersistentOfflineTopicStats(com.yahoo.pulsar.common.policies.data.PersistentOfflineTopicStats) ClusterData(com.yahoo.pulsar.common.policies.data.ClusterData) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Map(java.util.Map) ZooKeeperCacheListener(com.yahoo.pulsar.zookeeper.ZooKeeperCacheListener) PulsarService(com.yahoo.pulsar.broker.PulsarService) FieldParser(com.yahoo.pulsar.common.util.FieldParser) PulsarClientImpl(com.yahoo.pulsar.client.impl.PulsarClientImpl) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) URI(java.net.URI) PulsarClient(com.yahoo.pulsar.client.api.PulsarClient) SystemUtils(org.apache.commons.lang.SystemUtils) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) Set(java.util.Set) NioServerSocketChannel(io.netty.channel.socket.nio.NioServerSocketChannel) PooledByteBufAllocator(io.netty.buffer.PooledByteBufAllocator) InetSocketAddress(java.net.InetSocketAddress) Executors(java.util.concurrent.Executors) NioEventLoopGroup(io.netty.channel.nio.NioEventLoopGroup) ObjectMapperFactory(com.yahoo.pulsar.common.util.ObjectMapperFactory) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Metrics(com.yahoo.pulsar.broker.stats.Metrics) List(java.util.List) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) ClientConfiguration(com.yahoo.pulsar.client.api.ClientConfiguration) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) Optional(java.util.Optional) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ChannelOption(io.netty.channel.ChannelOption) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) EpollChannelOption(io.netty.channel.epoll.EpollChannelOption) Lists(com.google.common.collect.Lists) FieldContext(com.yahoo.pulsar.common.configuration.FieldContext) ByteBuf(io.netty.buffer.ByteBuf) EpollServerSocketChannel(io.netty.channel.epoll.EpollServerSocketChannel) RetentionPolicies(com.yahoo.pulsar.common.policies.data.RetentionPolicies) ConcurrentOpenHashSet(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) EpollEventLoopGroup(io.netty.channel.epoll.EpollEventLoopGroup) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) PersistentReplicator(com.yahoo.pulsar.broker.service.persistent.PersistentReplicator) ClusterReplicationMetrics(com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics) Logger(org.slf4j.Logger) ZooKeeperDataCache(com.yahoo.pulsar.zookeeper.ZooKeeperDataCache) EventLoopGroup(io.netty.channel.EventLoopGroup) KeeperException(org.apache.zookeeper.KeeperException) Semaphore(java.util.concurrent.Semaphore) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) IOException(java.io.IOException) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) Field(java.lang.reflect.Field) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarWebResource(com.yahoo.pulsar.broker.web.PulsarWebResource) Closeable(java.io.Closeable) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationManager(com.yahoo.pulsar.broker.authorization.AuthorizationManager) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) AuthenticationService(com.yahoo.pulsar.broker.authentication.AuthenticationService) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ServiceUnitNotReadyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback)

Example 5 with PersistenceException

use of com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException in project pulsar by yahoo.

the class PersistentTopic method addFailed.

@Override
public void addFailed(ManagedLedgerException exception, Object ctx) {
    PublishCallback callback = (PublishCallback) ctx;
    log.error("[{}] Failed to persist msg in store: {}", topic, exception.getMessage());
    callback.completed(new PersistenceException(exception), -1, -1);
    if (exception instanceof ManagedLedgerFencedException) {
        // If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen
        close();
    }
}
Also used : PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException)

Aggregations

PersistenceException (com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException)8 CompletableFuture (java.util.concurrent.CompletableFuture)7 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)7 DeleteCursorCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback)4 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 AdminResource (com.yahoo.pulsar.broker.admin.AdminResource)3 ServerMetadataException (com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException)3 TopicBusyException (com.yahoo.pulsar.broker.service.BrokerServiceException.TopicBusyException)3 TopicFencedException (com.yahoo.pulsar.broker.service.BrokerServiceException.TopicFencedException)3 ClusterReplicationMetrics (com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics)3 FutureUtil (com.yahoo.pulsar.client.util.FutureUtil)3 DestinationName (com.yahoo.pulsar.common.naming.DestinationName)3 PersistentTopicStats (com.yahoo.pulsar.common.policies.data.PersistentTopicStats)3 Policies (com.yahoo.pulsar.common.policies.data.Policies)3 Lists (com.beust.jcommander.internal.Lists)2 ObjectObjectHashMap (com.carrotsearch.hppc.ObjectObjectHashMap)2 Objects (com.google.common.base.Objects)2 Maps (com.google.common.collect.Maps)2 Sets (com.google.common.collect.Sets)2 BrokerService (com.yahoo.pulsar.broker.service.BrokerService)2