Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 71 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class ManagedLedgerOfflineBacklog method readLedgerMeta.

private void readLedgerMeta(final ManagedLedgerFactoryImpl factory, final TopicName topicName, final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers) throws Exception {
    String managedLedgerName = topicName.getPersistenceNamingEncoding();
    MetaStore store = factory.getMetaStore();
    BookKeeper bk = factory.getBookKeeper();
    final CountDownLatch mlMetaCounter = new CountDownLatch(1);
    store.getManagedLedgerInfo(managedLedgerName, new MetaStore.MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {

        @Override
        public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, MetaStore.Stat version) {
            for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : mlInfo.getLedgerInfoList()) {
                ledgers.put(ls.getLedgerId(), ls);
            }
            // find no of entries in last ledger
            if (ledgers.size() > 0) {
                final long id = ledgers.lastKey();
                AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Opened ledger {}: ", managedLedgerName, id, BKException.getMessage(rc));
                    }
                    if (rc == BKException.Code.OK) {
                        MLDataFormats.ManagedLedgerInfo.LedgerInfo info = MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(id).setEntries(lh.getLastAddConfirmed() + 1).setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build();
                        ledgers.put(id, info);
                        mlMetaCounter.countDown();
                    } else if (rc == BKException.Code.NoSuchLedgerExistsException) {
                        log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey());
                        ledgers.remove(ledgers.lastKey());
                        mlMetaCounter.countDown();
                    } else {
                        log.error("[{}] Failed to open ledger {}: {}", managedLedgerName, id, BKException.getMessage(rc));
                        mlMetaCounter.countDown();
                    }
                };
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Opening ledger {}", managedLedgerName, id);
                }
                try {
                    bk.asyncOpenLedgerNoRecovery(id, digestType, password, opencb, null);
                } catch (Exception e) {
                    log.warn("[{}] Failed to open ledger {}: {}", managedLedgerName, id, e);
                    mlMetaCounter.countDown();
                }
            } else {
                log.warn("[{}] Ledger list empty", managedLedgerName);
                mlMetaCounter.countDown();
            }
        }

        @Override
        public void operationFailed(ManagedLedgerException.MetaStoreException e) {
            log.warn("[{}] Unable to obtain managed ledger metadata - {}", e);
            mlMetaCounter.countDown();
        }
    });
    if (accurate) {
        // block until however long it takes for operation to complete
        mlMetaCounter.await();
    } else {
        mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
    }
}

Example 72 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class CompactedTopicTest method testCleanupOldCompactedTopicLedger.

@Test
public void testCleanupOldCompactedTopicLedger() throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null);
    LedgerHandle oldCompactedLedger = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    oldCompactedLedger.close();
    LedgerHandle newCompactedLedger = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    newCompactedLedger.close();
    // set the compacted topic ledger
    CompactedTopicImpl compactedTopic = new CompactedTopicImpl(bk);
    compactedTopic.newCompactedLedger(new PositionImpl(1, 2), oldCompactedLedger.getId()).get();
    // ensure both ledgers still exist, can be opened
    bk.openLedger(oldCompactedLedger.getId(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
    bk.openLedger(newCompactedLedger.getId(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
    // update the compacted topic ledger
    compactedTopic.newCompactedLedger(new PositionImpl(1, 2), newCompactedLedger.getId()).get();
    // old ledger should be deleted, new still there
    try {
        bk.openLedger(oldCompactedLedger.getId(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
        Assert.fail("Should have failed to open old ledger");
    } catch (BKException.BKNoSuchLedgerExistsException e) {
    // correct, expected behaviour
    }
    bk.openLedger(newCompactedLedger.getId(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
}

Example 73 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class CompactorTest method compactAndVerify.

private List<String> compactAndVerify(String topic, Map<String, byte[]> expected) throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null);
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    long compactedLedgerId = compactor.compact(topic).get();
    LedgerHandle ledger = bk.openLedger(compactedLedgerId, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    // 0..lac
    Assert.assertEquals(// 0..lac
    ledger.getLastAddConfirmed() + 1, expected.size(), "Should have as many entries as there is keys");
    List<String> keys = new ArrayList<>();
    Enumeration<LedgerEntry> entries = ledger.readEntries(0, ledger.getLastAddConfirmed());
    while (entries.hasMoreElements()) {
        ByteBuf buf = entries.nextElement().getEntryBuffer();
        RawMessage m = RawMessageImpl.deserializeFrom(buf);
        String key = extractKey(m);
        keys.add(key);
        ByteBuf payload = extractPayload(m);
        byte[] bytes = new byte[payload.readableBytes()];
        payload.readBytes(bytes);
        Assert.assertEquals(bytes, expected.remove(key), "Compacted version should match expected version");
        m.close();
    }
    Assert.assertTrue(expected.isEmpty(), "All expected keys should have been found");
    return keys;
}

Example 74 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class TwoPhaseCompactor method phaseTwoSeekThenLoop.

private CompletableFuture<Long> phaseTwoSeekThenLoop(RawReader reader, MessageId from, MessageId to, Map<String, MessageId> latestForKey, BookKeeper bk, LedgerHandle ledger) {
    CompletableFuture<Long> promise = new CompletableFuture<>();
    reader.seekAsync(from).thenCompose((v) -> {
        Semaphore outstanding = new Semaphore(MAX_OUTSTANDING);
        CompletableFuture<Void> loopPromise = new CompletableFuture<Void>();
        phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise);
        return loopPromise;
    }).thenCompose((v) -> closeLedger(ledger)).thenCompose((v) -> reader.acknowledgeCumulativeAsync(to, ImmutableMap.of(COMPACTED_TOPIC_LEDGER_PROPERTY, ledger.getId()))).whenComplete((res, exception) -> {
        if (exception != null) {
            deleteLedger(bk, ledger).whenComplete((res2, exception2) -> {
                if (exception2 != null) {
                    log.warn("Cleanup of ledger {} for failed", ledger, exception2);
                }
                // complete with original exception
                promise.completeExceptionally(exception);
            });
        } else {
            promise.complete(ledger.getId());
        }
    });
    return promise;
}

Example 75 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project herddb by diennea.

the class LedgerClosedTest method uselessLedgerDroppedError.

@Test
public void uselessLedgerDroppedError() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 0);
    }
    // restart
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
    }
    // restart
    LedgersInfo actualLedgersList;
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        actualLedgersList = log.getActualLedgersList();
        assertEquals(3, actualLedgersList.getActiveLedgers().size());
        System.out.println("actualLedgersList: " + actualLedgersList);
        long lastLedgerId = log.getLastLedgerId();
        assertEquals(lastLedgerId, actualLedgersList.getActiveLedgers().get(actualLedgersList.getActiveLedgers().size() - 1).longValue());
        try (BookKeeper bk = createBookKeeper()) {
            long ledgerIdToDrop = actualLedgersList.getActiveLedgers().get(0);
            System.out.println("dropping " + ledgerIdToDrop);
            bk.newDeleteLedgerOp().withLedgerId(ledgerIdToDrop).execute().get();
        }
    }
    // the server should boot even if the ledger does not exist anymore
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
    }
}