Search in sources :

Example 96 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class PersistentTopic method terminate.

public CompletableFuture<MessageId> terminate() {
    CompletableFuture<MessageId> future = new CompletableFuture<>();
    ledger.asyncTerminate(new TerminateCallback() {

        @Override
        public void terminateComplete(Position lastCommittedPosition, Object ctx) {
            producers.forEach(Producer::disconnect);
            subscriptions.forEach((name, sub) -> sub.topicTerminated());
            PositionImpl lastPosition = (PositionImpl) lastCommittedPosition;
            MessageId messageId = new MessageIdImpl(lastPosition.getLedgerId(), lastPosition.getEntryId(), -1);
            log.info("[{}] Topic terminated at {}", getName(), messageId);
            future.complete(messageId);
        }

        @Override
        public void terminateFailed(ManagedLedgerException exception, Object ctx) {
            future.completeExceptionally(exception);
        }
    }, null);
    return future;
}
Also used : SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) LedgerInfo(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats.LedgerInfo) PersistentTopicStats(org.apache.pulsar.common.policies.data.PersistentTopicStats) AdminResource(org.apache.pulsar.broker.admin.AdminResource) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) SchemaVersion(org.apache.pulsar.common.schema.SchemaVersion) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) ProducerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ProducerBusyException) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) Map(java.util.Map) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) BrokerService(org.apache.pulsar.broker.service.BrokerService) ReplicatorStats(org.apache.pulsar.common.policies.data.ReplicatorStats) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) Sets(com.google.common.collect.Sets) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) ServerCnx(org.apache.pulsar.broker.service.ServerCnx) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) SchemaData(org.apache.pulsar.common.schema.SchemaData) Optional(java.util.Optional) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Subscription(org.apache.pulsar.broker.service.Subscription) TopicName(org.apache.pulsar.common.naming.TopicName) CursorStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) DateFormatter(org.apache.pulsar.common.util.DateFormatter) Lists(com.google.common.collect.Lists) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) Logger(org.slf4j.Logger) SubType(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType) KeeperException(org.apache.zookeeper.KeeperException) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) MoreObjects(com.google.common.base.MoreObjects) AtomicLongFieldUpdater(java.util.concurrent.atomic.AtomicLongFieldUpdater) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) Compactor(org.apache.pulsar.compaction.Compactor) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) POLICIES(org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES) Codec(org.apache.pulsar.common.util.Codec) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) InitialPosition(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) Position(org.apache.bookkeeper.mledger.Position) InitialPosition(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageId(org.apache.pulsar.client.api.MessageId)

Example 97 with Position

use of org.apache.bookkeeper.mledger.Position in project pulsar by yahoo.

the class ManagedCursorTest method testFindNewestMatchingEdgeCase3.

@Test(timeOut = 20000)
void testFindNewestMatchingEdgeCase3() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    Position p1 = ledger.addEntry("expired".getBytes(Encoding));
    ledger.addEntry("not-expired".getBytes(Encoding));
    assertEquals(c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))), p1);
}
Also used : ZNodeProtobufFormat(org.apache.bookkeeper.mledger.impl.MetaStoreImplZookeeper.ZNodeProtobufFormat) Arrays(java.util.Arrays) Assert.assertNull(org.testng.Assert.assertNull) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) MockedBookKeeperTestCase(org.apache.bookkeeper.test.MockedBookKeeperTestCase) Entry(org.apache.bookkeeper.mledger.Entry) LoggerFactory(org.slf4j.LoggerFactory) Assert.assertEquals(org.testng.Assert.assertEquals) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) Test(org.testng.annotations.Test) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Lists(com.google.common.collect.Lists) Charset(java.nio.charset.Charset) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) Assert.assertFalse(org.testng.Assert.assertFalse) ExecutorService(java.util.concurrent.ExecutorService) Charsets(com.google.common.base.Charsets) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) CyclicBarrier(java.util.concurrent.CyclicBarrier) Logger(org.slf4j.Logger) Factory(org.testng.annotations.Factory) Assert.fail(org.testng.Assert.fail) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) BKException(org.apache.bookkeeper.client.BKException) Sets(com.google.common.collect.Sets) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) TimeUnit(java.util.concurrent.TimeUnit) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) MarkDeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback) Assert.assertTrue(org.testng.Assert.assertTrue) Collections(java.util.Collections) ReadEntriesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) Test(org.testng.annotations.Test)

Example 98 with Position

use of org.apache.bookkeeper.mledger.Position in project pulsar by yahoo.

the class ManagedLedgerBinaryFormatConversion method textToBinary.

@Test(timeOut = 20000, dataProvider = "gracefulClose")
void textToBinary(boolean gracefulClose) throws Exception {
    ManagedLedgerFactoryConfig textConf = new ManagedLedgerFactoryConfig();
    textConf.setUseProtobufBinaryFormatInZK(false);
    ManagedLedgerFactory textFactory = new ManagedLedgerFactoryImpl(bkc, zkc, textConf);
    ManagedLedger ledger = textFactory.open("my_test_ledger");
    ManagedCursor c1 = ledger.openCursor("c1");
    ledger.addEntry("test-0".getBytes(Encoding));
    Position p1 = ledger.addEntry("test-1".getBytes(Encoding));
    ledger.addEntry("test-2".getBytes(Encoding));
    c1.delete(p1);
    if (gracefulClose) {
        ledger.close();
    }
    // Reopen with binary format
    ManagedLedgerFactoryConfig binaryConf = new ManagedLedgerFactoryConfig();
    binaryConf.setUseProtobufBinaryFormatInZK(true);
    ManagedLedgerFactory binaryFactory = new ManagedLedgerFactoryImpl(bkc, zkc, binaryConf);
    ledger = binaryFactory.open("my_test_ledger");
    c1 = ledger.openCursor("c1");
    // The 'p1' entry was already deleted
    assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
    textFactory.shutdown();
    binaryFactory.shutdown();
}
Also used : ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) Position(org.apache.bookkeeper.mledger.Position) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 99 with Position

use of org.apache.bookkeeper.mledger.Position in project pulsar by yahoo.

the class ManagedLedgerBinaryFormatConversion method binaryToText.

@Test(timeOut = 20000, dataProvider = "gracefulClose")
void binaryToText(boolean gracefulClose) throws Exception {
    ManagedLedgerFactoryConfig binaryConf = new ManagedLedgerFactoryConfig();
    binaryConf.setUseProtobufBinaryFormatInZK(true);
    ManagedLedgerFactory binaryFactory = new ManagedLedgerFactoryImpl(bkc, zkc, binaryConf);
    ManagedLedger ledger = binaryFactory.open("my_test_ledger");
    ManagedCursor c1 = ledger.openCursor("c1");
    ledger.addEntry("test-0".getBytes(Encoding));
    Position p1 = ledger.addEntry("test-1".getBytes(Encoding));
    ledger.addEntry("test-2".getBytes(Encoding));
    c1.delete(p1);
    if (gracefulClose) {
        ledger.close();
    }
    // Reopen with binary format
    ManagedLedgerFactoryConfig textConf = new ManagedLedgerFactoryConfig();
    textConf.setUseProtobufBinaryFormatInZK(false);
    ManagedLedgerFactory textFactory = new ManagedLedgerFactoryImpl(bkc, zkc, textConf);
    ledger = textFactory.open("my_test_ledger");
    c1 = ledger.openCursor("c1");
    // The 'p1' entry was already deleted
    assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
    textFactory.shutdown();
    binaryFactory.shutdown();
}
Also used : ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) Position(org.apache.bookkeeper.mledger.Position) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 100 with Position

use of org.apache.bookkeeper.mledger.Position in project pulsar by yahoo.

the class ManagedLedgerBkTest method asyncMarkDeleteAndClose.

@Test
public void asyncMarkDeleteAndClose() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig config = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1).setMetadataAckQuorumSize(1);
    ManagedLedger ledger = factory.open("my_test_ledger", config);
    ManagedCursor cursor = ledger.openCursor("c1");
    List<Position> positions = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        Position p = ledger.addEntry("entry".getBytes());
        positions.add(p);
    }
    final CountDownLatch counter = new CountDownLatch(positions.size());
    final AtomicBoolean gotException = new AtomicBoolean(false);
    for (Position p : positions) {
        cursor.asyncDelete(p, new DeleteCallback() {

            @Override
            public void deleteComplete(Object ctx) {
                // Ok
                counter.countDown();
            }

            @Override
            public void deleteFailed(ManagedLedgerException exception, Object ctx) {
                exception.printStackTrace();
                gotException.set(true);
                counter.countDown();
            }
        }, null);
    }
    cursor.close();
    ledger.close();
    counter.await();
    assertFalse(gotException.get());
    factory.shutdown();
}
Also used : Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) CountDownLatch(java.util.concurrent.CountDownLatch) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) DeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Test(org.testng.annotations.Test)

Aggregations

Position (org.apache.bookkeeper.mledger.Position)201 Test (org.testng.annotations.Test)169 ManagedLedger (org.apache.bookkeeper.mledger.ManagedLedger)168 ManagedCursor (org.apache.bookkeeper.mledger.ManagedCursor)167 ManagedLedgerConfig (org.apache.bookkeeper.mledger.ManagedLedgerConfig)127 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)115 Entry (org.apache.bookkeeper.mledger.Entry)104 CountDownLatch (java.util.concurrent.CountDownLatch)97 AddEntryCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)72 ManagedLedgerFactory (org.apache.bookkeeper.mledger.ManagedLedgerFactory)68 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)58 BKException (org.apache.bookkeeper.client.BKException)56 List (java.util.List)55 MarkDeleteCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback)53 AtomicReference (java.util.concurrent.atomic.AtomicReference)52 AsyncCallbacks (org.apache.bookkeeper.mledger.AsyncCallbacks)52 Logger (org.slf4j.Logger)52 LoggerFactory (org.slf4j.LoggerFactory)52 Lists (com.google.common.collect.Lists)51 TimeUnit (java.util.concurrent.TimeUnit)51