Search in sources :

Example 1 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class ManagedLedgerTest method testBackwardCompatiblityForMeta.

@Test
public void testBackwardCompatiblityForMeta() throws Exception {
    final ManagedLedgerInfo[] storedMLInfo = new ManagedLedgerInfo[3];
    final Stat[] versions = new Stat[1];
    ManagedLedgerConfig conf = new ManagedLedgerConfig();
    conf.setMaxEntriesPerLedger(1);
    conf.setRetentionSizeInMB(10);
    conf.setRetentionTime(1, TimeUnit.HOURS);
    ManagedLedger ml = factory.open("backward_test_ledger", conf);
    ml.openCursor("c1");
    ml.addEntry("msg1".getBytes());
    ml.addEntry("msg2".getBytes());
    ml.close();
    MetaStore store = new MetaStoreImpl(metadataStore, executor);
    CountDownLatch l1 = new CountDownLatch(1);
    // obtain the ledger info
    store.getManagedLedgerInfo("backward_test_ledger", false, new MetaStoreCallback<ManagedLedgerInfo>() {

        @Override
        public void operationComplete(ManagedLedgerInfo result, Stat version) {
            storedMLInfo[0] = result;
            versions[0] = version;
            l1.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            fail("on get ManagedLedgerInfo backward_test_ledger");
        }
    });
    l1.await();
    ManagedLedgerInfo.Builder builder1 = ManagedLedgerInfo.newBuilder();
    // simulate test for old ledger with no timestampl
    for (LedgerInfo info : storedMLInfo[0].getLedgerInfoList()) {
        LedgerInfo noTimestamp = ManagedLedgerInfo.LedgerInfo.newBuilder().mergeFrom(info).clearTimestamp().build();
        assertFalse(noTimestamp.hasTimestamp(), "expected old version info with no timestamp");
        builder1.addLedgerInfo(noTimestamp);
    }
    storedMLInfo[1] = builder1.build();
    // test timestamp on new ledger
    CountDownLatch l2 = new CountDownLatch(1);
    store.asyncUpdateLedgerIds("backward_test_ledger", storedMLInfo[1], versions[0], new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void result, Stat version) {
            l2.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            fail("on asyncUpdateLedgerIds");
        }
    });
    // verify that after update ledgers have timestamp
    ManagedLedgerImpl newVersionLedger = (ManagedLedgerImpl) factory.open("backward_test_ledger", conf);
    List<LedgerInfo> mlInfo = newVersionLedger.getLedgersInfoAsList();
    assertTrue(mlInfo.stream().allMatch(ledgerInfo -> ledgerInfo.hasTimestamp()));
}
Also used : Arrays(java.util.Arrays) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) Cleanup(lombok.Cleanup) LedgerOffloader(org.apache.bookkeeper.mledger.LedgerOffloader) Test(org.testng.annotations.Test) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) GeneralSecurityException(java.security.GeneralSecurityException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Pair(org.apache.commons.lang3.tuple.Pair) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) EnsemblePlacementPolicy(org.apache.bookkeeper.client.EnsemblePlacementPolicy) Mockito.doReturn(org.mockito.Mockito.doReturn) Assert.assertFalse(org.testng.Assert.assertFalse) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) Stat(org.apache.pulsar.metadata.api.Stat) AddCallback(org.apache.bookkeeper.client.AsyncCallback.AddCallback) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) Set(java.util.Set) Futures(org.apache.bookkeeper.mledger.util.Futures) Mockito.doNothing(org.mockito.Mockito.doNothing) MLDataFormats(org.apache.bookkeeper.mledger.proto.MLDataFormats) BookKeeper(org.apache.bookkeeper.client.BookKeeper) Executors(java.util.concurrent.Executors) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) CountDownLatch(java.util.concurrent.CountDownLatch) Slf4j(lombok.extern.slf4j.Slf4j) SessionEvent(org.apache.pulsar.metadata.api.extended.SessionEvent) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) Awaitility(org.awaitility.Awaitility) Mockito.mock(org.mockito.Mockito.mock) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) MockedBookKeeperTestCase(org.apache.bookkeeper.test.MockedBookKeeperTestCase) ReadEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback) Mockito.spy(org.mockito.Mockito.spy) ArrayList(java.util.ArrayList) EnsemblePlacementPolicyConfig(org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig) Assert(org.testng.Assert) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) Charsets(com.google.common.base.Charsets) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) Mockito.times(org.mockito.Mockito.times) Field(java.lang.reflect.Field) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) FaultInjectionMetadataStore(org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore) PulsarMockLedgerHandle(org.apache.bookkeeper.client.PulsarMockLedgerHandle) ClientConfiguration(org.apache.bookkeeper.conf.ClientConfiguration) MarkDeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) ReadOnlyBufferException(java.nio.ReadOnlyBufferException) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) ScheduledFuture(java.util.concurrent.ScheduledFuture) PulsarMockBookKeeper(org.apache.bookkeeper.client.PulsarMockBookKeeper) ByteBufAllocator(io.netty.buffer.ByteBufAllocator) TimeoutException(java.util.concurrent.TimeoutException) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) MutableObject(org.apache.commons.lang3.mutable.MutableObject) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) Assert.assertSame(org.testng.Assert.assertSame) CyclicBarrier(java.util.concurrent.CyclicBarrier) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) Predicate(java.util.function.Predicate) OffloadPoliciesImpl(org.apache.pulsar.common.policies.data.OffloadPoliciesImpl) Assert.assertNotNull(org.testng.Assert.assertNotNull) UUID(java.util.UUID) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) Sets(com.google.common.collect.Sets) BKException(org.apache.bookkeeper.client.BKException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) List(java.util.List) Optional(java.util.Optional) LedgerEntries(org.apache.bookkeeper.client.api.LedgerEntries) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) Assert.assertNull(org.testng.Assert.assertNull) DataProvider(org.testng.annotations.DataProvider) Entry(org.apache.bookkeeper.mledger.Entry) Assert.assertEquals(org.testng.Assert.assertEquals) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ByteBuf(io.netty.buffer.ByteBuf) Charset(java.nio.charset.Charset) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) ExecutorService(java.util.concurrent.ExecutorService) Iterator(java.util.Iterator) Assert.fail(org.testng.Assert.fail) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) ReadHandle(org.apache.bookkeeper.client.api.ReadHandle) Mockito(org.mockito.Mockito) VoidCallback(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback) Assert.assertTrue(org.testng.Assert.assertTrue) Collections(java.util.Collections) ReadEntriesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) CountDownLatch(java.util.concurrent.CountDownLatch) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) LedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Stat(org.apache.pulsar.metadata.api.Stat) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Test(org.testng.annotations.Test)

Example 2 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class MetaStoreImplTest method failInCreatingMLnode.

@Test(timeOut = 20000)
void failInCreatingMLnode() throws Exception {
    MetaStore store = new MetaStoreImpl(metadataStore, executor);
    final CompletableFuture<Void> promise = new CompletableFuture<>();
    metadataStore.failConditional(new MetadataStoreException("error"), (op, path) -> op == FaultInjectionMetadataStore.OperationType.PUT);
    store.getManagedLedgerInfo("my_test", false, new MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {

        public void operationFailed(MetaStoreException e) {
            promise.complete(null);
        }

        public void operationComplete(ManagedLedgerInfo result, Stat version) {
            promise.completeExceptionally(new Exception("Operation should have failed"));
        }
    });
    promise.get();
}
Also used : MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) CompletableFuture(java.util.concurrent.CompletableFuture) Stat(org.apache.pulsar.metadata.api.Stat) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) Test(org.testng.annotations.Test)

Example 3 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class MetaStoreImplTest method updatingMLNode.

@Test(timeOut = 20000)
void updatingMLNode() throws Exception {
    MetaStore store = new MetaStoreImpl(metadataStore, executor);
    metadataStore.put("/managed-ledgers/my_test", "".getBytes(), Optional.empty());
    final CompletableFuture<Void> promise = new CompletableFuture<>();
    store.getManagedLedgerInfo("my_test", false, new MetaStoreCallback<ManagedLedgerInfo>() {

        public void operationFailed(MetaStoreException e) {
            promise.completeExceptionally(e);
        }

        public void operationComplete(ManagedLedgerInfo mlInfo, Stat version) {
            // Update again using the version
            metadataStore.failConditional(new MetadataStoreException.BadVersionException("error"), (op, path) -> op == FaultInjectionMetadataStore.OperationType.PUT && path.contains("my_test"));
            store.asyncUpdateLedgerIds("my_test", mlInfo, version, new MetaStoreCallback<Void>() {

                public void operationFailed(MetaStoreException e) {
                    // ok
                    promise.complete(null);
                }

                @Override
                public void operationComplete(Void result, Stat version) {
                    promise.completeExceptionally(new Exception("should have failed"));
                }
            });
        }
    });
    promise.get();
}
Also used : ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) Stat(org.apache.pulsar.metadata.api.Stat) MockedBookKeeperTestCase(org.apache.bookkeeper.test.MockedBookKeeperTestCase) Assert.fail(org.testng.Assert.fail) CompletableFuture(java.util.concurrent.CompletableFuture) Test(org.testng.annotations.Test) Assert.assertNotNull(org.testng.Assert.assertNotNull) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) MLDataFormats(org.apache.bookkeeper.mledger.proto.MLDataFormats) AtomicReference(java.util.concurrent.atomic.AtomicReference) ManagedCursorInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo) CountDownLatch(java.util.concurrent.CountDownLatch) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) FaultInjectionMetadataStore(org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore) MetadataCache(org.apache.pulsar.metadata.api.MetadataCache) Data(lombok.Data) Assert.assertTrue(org.testng.Assert.assertTrue) Optional(java.util.Optional) AllArgsConstructor(lombok.AllArgsConstructor) Assert.assertFalse(org.testng.Assert.assertFalse) NoArgsConstructor(lombok.NoArgsConstructor) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) CompletableFuture(java.util.concurrent.CompletableFuture) Stat(org.apache.pulsar.metadata.api.Stat) Test(org.testng.annotations.Test)

Example 4 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class MetaStoreImplTest method deleteNonExistingML.

@Test
void deleteNonExistingML() throws Exception {
    MetaStore store = new MetaStoreImpl(metadataStore, executor);
    AtomicReference<MetaStoreException> exception = new AtomicReference<>();
    CountDownLatch counter = new CountDownLatch(1);
    store.removeManagedLedger("non-existing", new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void result, Stat version) {
            counter.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            exception.set(e);
            counter.countDown();
        }
    });
    counter.await();
    assertNotNull(exception.get());
}
Also used : MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Stat(org.apache.pulsar.metadata.api.Stat) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 5 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class MetaStoreImplTest method readMalformedML.

@Test(timeOut = 20000)
void readMalformedML() throws Exception {
    MetaStore store = new MetaStoreImpl(metadataStore, executor);
    metadataStore.put("/managed-ledgers/my_test", "non-valid".getBytes(), Optional.empty()).join();
    final CountDownLatch latch = new CountDownLatch(1);
    store.getManagedLedgerInfo("my_test", false, new MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {

        public void operationFailed(MetaStoreException e) {
            // Ok
            latch.countDown();
        }

        public void operationComplete(ManagedLedgerInfo result, Stat version) {
            fail("Operation should have failed");
        }
    });
    latch.await();
}
Also used : MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Stat(org.apache.pulsar.metadata.api.Stat) CountDownLatch(java.util.concurrent.CountDownLatch) ManagedLedgerInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo) Test(org.testng.annotations.Test)

Aggregations

Stat (org.apache.pulsar.metadata.api.Stat)44 MetaStoreException (org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException)23 Test (org.testng.annotations.Test)22 CompletableFuture (java.util.concurrent.CompletableFuture)14 CountDownLatch (java.util.concurrent.CountDownLatch)14 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)13 ManagedLedgerInfo (org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo)12 Cleanup (lombok.Cleanup)11 MetadataStoreException (org.apache.pulsar.metadata.api.MetadataStoreException)11 MetaStoreCallback (org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback)10 MetadataStore (org.apache.pulsar.metadata.api.MetadataStore)9 Optional (java.util.Optional)8 ManagedCursorInfo (org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo)8 MLDataFormats (org.apache.bookkeeper.mledger.proto.MLDataFormats)7 ArrayList (java.util.ArrayList)6 Slf4j (lombok.extern.slf4j.Slf4j)6 BKException (org.apache.bookkeeper.client.BKException)6 BookKeeper (org.apache.bookkeeper.client.BookKeeper)6 LedgerInfo (org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo)6 Notification (org.apache.pulsar.metadata.api.Notification)6