Search in sources :

Example 1 with OpenCursorCallback

use of org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback in project pulsar by yahoo.

the class ManagedLedgerImpl method openCursor.

@Override
public ManagedCursor openCursor(String cursorName) throws InterruptedException, ManagedLedgerException {
    final CountDownLatch counter = new CountDownLatch(1);
    class Result {

        ManagedCursor cursor = null;

        ManagedLedgerException exception = null;
    }
    final Result result = new Result();
    asyncOpenCursor(cursorName, new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            result.cursor = cursor;
            counter.countDown();
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            result.exception = exception;
            counter.countDown();
        }
    }, null);
    if (!counter.await(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS)) {
        throw new ManagedLedgerException("Timeout during open-cursor operation");
    }
    if (result.exception != null) {
        log.error("Error adding entry", result.exception);
        throw result.exception;
    }
    return result.cursor;
}
Also used : ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) CountDownLatch(java.util.concurrent.CountDownLatch) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor)

Example 2 with OpenCursorCallback

use of org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback in project pulsar by yahoo.

the class PersistentTopic method startReplicator.

CompletableFuture<Void> startReplicator(String remoteCluster) {
    log.info("[{}] Starting replicator to remote: {}", topic, remoteCluster);
    final CompletableFuture<Void> future = new CompletableFuture<>();
    String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster);
    ledger.asyncOpenCursor(name, new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
            replicators.computeIfAbsent(remoteCluster, r -> new PersistentReplicator(PersistentTopic.this, cursor, localCluster, remoteCluster, brokerService));
            future.complete(null);
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            future.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return future;
}
Also used : ReplicationMetrics(com.yahoo.pulsar.broker.stats.ReplicationMetrics) SubType(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType) PersistentSubscriptionStats(com.yahoo.pulsar.common.policies.data.PersistentSubscriptionStats) LedgerInfo(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.LedgerInfo) NamingException(com.yahoo.pulsar.broker.service.BrokerServiceException.NamingException) ConsumerStats(com.yahoo.pulsar.common.policies.data.ConsumerStats) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) PersistentTopicInternalStats(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats) LoggerFactory(org.slf4j.LoggerFactory) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) NamespaceBundleStats(com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats) Policies(com.yahoo.pulsar.common.policies.data.Policies) BacklogQuota(com.yahoo.pulsar.common.policies.data.BacklogQuota) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) TopicBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.TopicBusyException) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ReplicatorStats(com.yahoo.pulsar.common.policies.data.ReplicatorStats) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) Objects(com.google.common.base.Objects) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) CursorStats(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) Instant(java.time.Instant) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) TopicFencedException(com.yahoo.pulsar.broker.service.BrokerServiceException.TopicFencedException) Lists(com.beust.jcommander.internal.Lists) ZoneId(java.time.ZoneId) Sets(com.google.common.collect.Sets) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Topic(com.yahoo.pulsar.broker.service.Topic) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) Consumer(com.yahoo.pulsar.broker.service.Consumer) List(java.util.List) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) StatsOutputStream(com.yahoo.pulsar.utils.StatsOutputStream) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) UnsupportedVersionException(com.yahoo.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) SubscriptionBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) MessageImpl(com.yahoo.pulsar.client.impl.MessageImpl) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) ServerCnx(com.yahoo.pulsar.broker.service.ServerCnx) ByteBuf(io.netty.buffer.ByteBuf) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) ConcurrentOpenHashSet(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet) ConsumerBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) Codec(com.yahoo.pulsar.common.util.Codec) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) ClusterReplicationMetrics(com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) BrokerServiceException(com.yahoo.pulsar.broker.service.BrokerServiceException) PublisherStats(com.yahoo.pulsar.common.policies.data.PublisherStats) Producer(com.yahoo.pulsar.broker.service.Producer) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) AtomicLongFieldUpdater(java.util.concurrent.atomic.AtomicLongFieldUpdater) NamespaceStats(com.yahoo.pulsar.broker.stats.NamespaceStats) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) DateTimeFormatter(java.time.format.DateTimeFormatter) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor)

Example 3 with OpenCursorCallback

use of org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback in project pulsar by yahoo.

the class ServerCnxTest method setupMLAsyncCallbackMocks.

private void setupMLAsyncCallbackMocks() {
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    // call openLedgerComplete with ledgerMock on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerComplete(ledgerMock, null);
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*success.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call openLedgerFailed on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            new Thread(() -> {
                ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerFailed(new ManagedLedgerException("Managed ledger failure"), null);
            }).start();
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*fail.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call addComplete on ledger asyncAddEntry
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete(new PositionImpl(-1, -1), invocationOnMock.getArguments()[2]);
            return null;
        }
    }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
            return null;
        }
    }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorFailed(new ManagedLedgerException("Managed ledger failure"), null);
            return null;
        }
    }).when(ledgerMock).asyncOpenCursor(matches(".*fail.*"), any(OpenCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*success.*"), any(DeleteCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorFailed(new ManagedLedgerException("Managed ledger failure"), null);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*fail.*"), any(DeleteCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((CloseCallback) invocationOnMock.getArguments()[0]).closeComplete(null);
            return null;
        }
    }).when(cursorMock).asyncClose(any(CloseCallback.class), anyObject());
    doReturn(successSubName).when(cursorMock).getName();
}
Also used : PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ByteBuf(io.netty.buffer.ByteBuf) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Matchers.anyObject(org.mockito.Matchers.anyObject) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)

Example 4 with OpenCursorCallback

use of org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback in project pulsar by yahoo.

the class ManagedLedgerTest method testConcurrentOpenCursor.

@Test
public void testConcurrentOpenCursor() throws Exception {
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testConcurrentOpenCursor");
    final AtomicReference<ManagedCursor> cursor1 = new AtomicReference<>(null);
    final AtomicReference<ManagedCursor> cursor2 = new AtomicReference<>(null);
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch latch = new CountDownLatch(2);
    cachedExecutor.execute(() -> {
        try {
            barrier.await();
        } catch (Exception e) {
        }
        ledger.asyncOpenCursor("c1", new OpenCursorCallback() {

            @Override
            public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
                latch.countDown();
            }

            @Override
            public void openCursorComplete(ManagedCursor cursor, Object ctx) {
                cursor1.set(cursor);
                latch.countDown();
            }
        }, null);
    });
    cachedExecutor.execute(() -> {
        try {
            barrier.await();
        } catch (Exception e) {
        }
        ledger.asyncOpenCursor("c1", new OpenCursorCallback() {

            @Override
            public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
                latch.countDown();
            }

            @Override
            public void openCursorComplete(ManagedCursor cursor, Object ctx) {
                cursor2.set(cursor);
                latch.countDown();
            }
        }, null);
    });
    latch.await();
    assertNotNull(cursor1.get());
    assertNotNull(cursor2.get());
    assertEquals(cursor1.get(), cursor2.get());
    ledger.close();
}
Also used : OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) BKException(org.apache.bookkeeper.client.BKException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) IOException(java.io.IOException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) CyclicBarrier(java.util.concurrent.CyclicBarrier) Test(org.testng.annotations.Test)

Example 5 with OpenCursorCallback

use of org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback in project pulsar by yahoo.

the class PersistentTopicTest method setupMLAsyncCallbackMocks.

void setupMLAsyncCallbackMocks() {
    ledgerMock = mock(ManagedLedger.class);
    cursorMock = mock(ManagedCursor.class);
    final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    doReturn("mockCursor").when(cursorMock).getName();
    // doNothing().when(cursorMock).asyncClose(new CloseCallback() {
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            // return closeFuture.get();
            return closeFuture.complete(null);
        }
    }).when(cursorMock).asyncClose(new CloseCallback() {

        @Override
        public void closeComplete(Object ctx) {
            log.info("[{}] Successfully closed cursor ledger", "mockCursor");
            closeFuture.complete(null);
        }

        @Override
        public void closeFailed(ManagedLedgerException exception, Object ctx) {
            // isFenced.set(false);
            log.error("Error closing cursor for subscription", exception);
            closeFuture.completeExceptionally(new BrokerServiceException.PersistenceException(exception));
        }
    }, null);
    // call openLedgerComplete with ledgerMock on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerComplete(ledgerMock, null);
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*success.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call openLedgerFailed on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerFailed(new ManagedLedgerException("Managed ledger failure"), null);
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*fail.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call addComplete on ledger asyncAddEntry
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete(new PositionImpl(1, 1), invocationOnMock.getArguments()[2]);
            return null;
        }
    }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), anyObject());
    // call openCursorComplete on cursor asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
            return null;
        }
    }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
    // call deleteLedgerComplete on ledger asyncDelete
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteLedgerCallback) invocationOnMock.getArguments()[0]).deleteLedgerComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDelete(any(DeleteLedgerCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*success.*"), any(DeleteCursorCallback.class), anyObject());
}
Also used : ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) ByteBuf(io.netty.buffer.ByteBuf) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Answer(org.mockito.stubbing.Answer) CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Matchers.anyObject(org.mockito.Matchers.anyObject) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)

Aggregations

OpenCursorCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback)21 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)21 ManagedCursor (org.apache.bookkeeper.mledger.ManagedCursor)19 CountDownLatch (java.util.concurrent.CountDownLatch)12 ManagedLedger (org.apache.bookkeeper.mledger.ManagedLedger)12 AddEntryCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)10 CompletableFuture (java.util.concurrent.CompletableFuture)9 DeleteCursorCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback)9 ByteBuf (io.netty.buffer.ByteBuf)8 CloseCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback)8 ManagedLedgerConfig (org.apache.bookkeeper.mledger.ManagedLedgerConfig)8 Test (org.testng.annotations.Test)8 List (java.util.List)6 Entry (org.apache.bookkeeper.mledger.Entry)6 ManagedLedgerFencedException (org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException)6 Position (org.apache.bookkeeper.mledger.Position)6 OpenLedgerCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback)5 PositionImpl (org.apache.bookkeeper.mledger.impl.PositionImpl)5 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)4 Maps (com.google.common.collect.Maps)4