Examples with ManagedCursor org.apache.bookkeeper.mledger.ManagedCursor used on opensource projects

Example 26 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class ManagedCursorTest method testRateLimitMarkDelete.

@Test(timeOut = 20000)
void testRateLimitMarkDelete() throws Exception {
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    // Throttle to 1/s
    config.setThrottleMarkDelete(1);
    ManagedLedger ledger = factory.open("my_test_ledger", config);
    ManagedCursor c1 = ledger.openCursor("c1");
    Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
    c1.markDelete(p1);
    c1.markDelete(p2);
    c1.markDelete(p3);
    assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
    // Re-open to recover from storage
    ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig());
    c1 = ledger.openCursor("c1");
    // Only the 1st mark-delete was persisted
    assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
    factory2.shutdown();
}

Example 27 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class PersistentTopic method startReplicator.

CompletableFuture<Void> startReplicator(String remoteCluster) {
    log.info("[{}] Starting replicator to remote: {}", topic, remoteCluster);
    final CompletableFuture<Void> future = new CompletableFuture<>();
    String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster);
    ledger.asyncOpenCursor(name, new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
            replicators.computeIfAbsent(remoteCluster, r -> new PersistentReplicator(PersistentTopic.this, cursor, localCluster, remoteCluster, brokerService));
            future.complete(null);
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            future.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return future;
}

Example 28 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class ManagedLedgerBkTest method testConcurrentMarkDelete.

@Test
public void testConcurrentMarkDelete() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
    mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    // set the data ledger size
    mlConfig.setMaxEntriesPerLedger(100);
    // set the metadata ledger size to 1 to kick off many ledger switching cases
    mlConfig.setMetadataMaxEntriesPerLedger(10);
    ManagedLedger ledger = factory.open("ml-markdelete-ledger", mlConfig);
    final List<Position> addedEntries = Lists.newArrayList();
    int numCursors = 10;
    final CyclicBarrier barrier = new CyclicBarrier(numCursors);
    List<ManagedCursor> cursors = Lists.newArrayList();
    for (int i = 0; i < numCursors; i++) {
        cursors.add(ledger.openCursor(String.format("c%d", i)));
    }
    for (int i = 0; i < 50; i++) {
        Position pos = ledger.addEntry("entry".getBytes());
        addedEntries.add(pos);
    }
    List<Future<?>> futures = Lists.newArrayList();
    for (ManagedCursor cursor : cursors) {
        futures.add(executor.submit(() -> {
            barrier.await();
            for (Position position : addedEntries) {
                cursor.markDelete(position);
            }
            return null;
        }));
    }
    for (Future<?> future : futures) {
        future.get();
    }
    // Since in this test we roll-over the cursor ledger every 10 entries acknowledged, the background roll back
    // might still be happening when the futures are completed.
    Thread.sleep(1000);
    factory.shutdown();
}

Example 29 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class ManagedLedgerBkTest method verifyConcurrentUsage.

@Test
public void verifyConcurrentUsage() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(100 * 1024 * 1024);
    ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    ManagedLedgerConfig conf = new ManagedLedgerConfig();
    conf.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
    final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger", conf);
    int NumProducers = 1;
    int NumConsumers = 1;
    final AtomicBoolean done = new AtomicBoolean();
    final CyclicBarrier barrier = new CyclicBarrier(NumProducers + NumConsumers + 1);
    List<Future<?>> futures = Lists.newArrayList();
    for (int i = 0; i < NumProducers; i++) {
        futures.add(executor.submit(() -> {
            try {
                barrier.await();
                while (!done.get()) {
                    ledger.addEntry("entry".getBytes());
                    Thread.sleep(1);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }));
    }
    for (int i = 0; i < NumConsumers; i++) {
        final int idx = i;
        futures.add(executor.submit(() -> {
            try {
                barrier.await();
                ManagedCursor cursor = ledger.openCursor("my-cursor-" + idx);
                while (!done.get()) {
                    List<Entry> entries = cursor.readEntries(1);
                    if (!entries.isEmpty()) {
                        cursor.markDelete(entries.get(0).getPosition());
                    }
                    entries.forEach(e -> e.release());
                    Thread.sleep(2);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }));
    }
    barrier.await();
    Thread.sleep(1 * 1000);
    done.set(true);
    for (Future<?> future : futures) {
        future.get();
    }
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertTrue(cacheManager.mlFactoryMBean.getCacheHitsRate() > 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertTrue(cacheManager.mlFactoryMBean.getCacheHitsThroughput() > 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    factory.shutdown();
}

Example 30 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class ManagedLedgerBkTest method testOfflineTopicBacklog.

@Test
public void testOfflineTopicBacklog() throws Exception {
    ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
    factoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf);
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    ManagedLedger ledger = factory.open("property/cluster/namespace/my-ledger", config);
    ManagedCursor cursor = ledger.openCursor("c1");
    int N = 1;
    for (int i = 0; i < N; i++) {
        String entry = "entry-" + i;
        ledger.addEntry(entry.getBytes());
    }
    List<Entry> entries = cursor.readEntries(N);
    assertEquals(N, entries.size());
    entries.forEach(e -> e.release());
    ledger.close();
    ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(DigestType.CRC32, "".getBytes(Charsets.UTF_8), "", false);
    PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog((ManagedLedgerFactoryImpl) factory, "property/cluster/namespace/my-ledger");
    factory.shutdown();
    assertNotNull(offlineTopicStats);
}