Examples with WriteHandle org.apache.bookkeeper.client.api.WriteHandle used on opensource projects

Example 6 with WriteHandle

use of org.apache.bookkeeper.client.api.WriteHandle in project bookkeeper by apache.

the class TestMaxEnsembleChangeNum method testChangeEnsembleMaxNumWithWriter.

@Test
public void testChangeEnsembleMaxNumWithWriter() throws Exception {
    long lId;
    int numEntries = 5;
    int changeNum = 5;
    setBookkeeperConfig(new ClientConfiguration().setDelayEnsembleChange(false).setMaxAllowedEnsembleChanges(5));
    try (WriteHandle writer = result(newCreateLedgerOp().withAckQuorumSize(3).withWriteQuorumSize(3).withEnsembleSize(3).withPassword(password).execute())) {
        lId = writer.getId();
        // first fragment
        for (int i = 0; i < numEntries; i++) {
            writer.append(ByteBuffer.wrap(data));
        }
        assertEquals("There should be zero ensemble change", 1, getLedgerMetadata(lId).getEnsembles().size());
        simulateEnsembleChangeWithWriter(changeNum, numEntries, writer);
        // one more ensemble change
        startNewBookie();
        killBookie(writer.getLedgerMetadata().getEnsembleAt(writer.getLastAddConfirmed()).get(0));
        // add failure
        try {
            writer.append(ByteBuffer.wrap(data));
            fail("should not come to here");
        } catch (BKException exception) {
            assertEquals(exception.getCode(), WriteException);
        }
    }
}

Example 7 with WriteHandle

use of org.apache.bookkeeper.client.api.WriteHandle in project bookkeeper by apache.

the class SimpleTestCommand method run.

@Override
protected void run(BookKeeper bk) throws Exception {
    // test data
    byte[] data = new byte[100];
    try (WriteHandle wh = result(bk.newCreateLedgerOp().withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize).withAckQuorumSize(ackQuorumSize).withDigestType(DigestType.CRC32C).withPassword(new byte[0]).execute())) {
        System.out.println("Ledger ID: " + wh.getId());
        long lastReport = System.nanoTime();
        for (int i = 0; i < numEntries; i++) {
            wh.append(data);
            if (TimeUnit.SECONDS.convert(System.nanoTime() - lastReport, TimeUnit.NANOSECONDS) > 1) {
                System.out.println(i + " entries written");
                lastReport = System.nanoTime();
            }
        }
        System.out.println(numEntries + " entries written to ledger " + wh.getId());
    }
}

Example 8 with WriteHandle

use of org.apache.bookkeeper.client.api.WriteHandle in project pravega by pravega.

the class BookKeeperLog method rollover.

// endregion
// region Ledger Rollover
/**
 * Triggers an asynchronous rollover, if the current Write Ledger has exceeded its maximum length.
 * The rollover protocol is as follows:
 * 1. Create a new ledger.
 * 2. Create an in-memory copy of the metadata and add the new ledger to it.
 * 3. Update the metadata in ZooKeeper using compare-and-set.
 * 3.1 If the update fails, the newly created ledger is deleted and the operation stops.
 * 4. Swap in-memory pointers to the active Write Ledger (all future writes will go to the new ledger).
 * 5. Close the previous ledger (and implicitly seal it).
 * 5.1 If closing fails, there is nothing we can do. We've already opened a new ledger and new writes are going to it.
 *
 * NOTE: this method is not thread safe and is not meant to be executed concurrently. It should only be invoked as
 * part of the Rollover Processor.
 */
// Because this is an arg to SequentialAsyncProcessor, which wants a Runnable.
@SneakyThrows(DurableDataLogException.class)
private void rollover() {
    if (this.closed.get()) {
        // BookKeeperLog is closed; no point in running this.
        return;
    }
    long traceId = LoggerHelpers.traceEnterWithContext(log, this.traceObjectId, "rollover");
    val l = getWriteLedger().ledger;
    if (!l.isClosed() && l.getLength() < this.config.getBkLedgerMaxSize()) {
        // Nothing to do. Trigger the write processor just in case this rollover was invoked because the write
        // processor got a pointer to a LedgerHandle that was just closed by a previous run of the rollover processor.
        this.writeProcessor.runAsync();
        LoggerHelpers.traceLeave(log, this.traceObjectId, "rollover", traceId, false);
        return;
    }
    try {
        // Create new ledger.
        WriteHandle newLedger = Ledgers.create(this.bookKeeper, this.config, this.logId);
        log.debug("{}: Rollover: created new ledger {}.", this.traceObjectId, newLedger.getId());
        // Update the metadata.
        LogMetadata metadata = getLogMetadata();
        metadata = updateMetadata(metadata, newLedger, false);
        LedgerMetadata ledgerMetadata = metadata.getLedger(newLedger.getId());
        assert ledgerMetadata != null : "cannot find newly added ledger metadata";
        log.debug("{}: Rollover: updated metadata '{}.", this.traceObjectId, metadata);
        // Update pointers to the new ledger and metadata.
        WriteHandle oldLedger;
        synchronized (this.lock) {
            oldLedger = this.writeLedger.ledger;
            if (!oldLedger.isClosed()) {
                // Only mark the old ledger as Rolled Over if it is still open. Otherwise it means it was closed
                // because of some failure and should not be marked as such.
                this.writeLedger.setRolledOver(true);
            }
            this.writeLedger = new WriteLedger(newLedger, ledgerMetadata);
            this.logMetadata = metadata;
        }
        // Close the old ledger. This must be done outside of the lock, otherwise the pending writes (and their callbacks)
        // will be invoked within the lock, thus likely candidates for deadlocks.
        Ledgers.close(oldLedger);
        log.info("{}: Rollover: swapped ledger and metadata pointers (Old = {}, New = {}) and closed old ledger.", this.traceObjectId, oldLedger.getId(), newLedger.getId());
    } finally {
        // It's possible that we have writes in the queue that didn't get picked up because they exceeded the predicted
        // ledger length. Invoke the Write Processor to execute them.
        this.writeProcessor.runAsync();
        LoggerHelpers.traceLeave(log, this.traceObjectId, "rollover", traceId, true);
    }
}

Example 9 with WriteHandle

use of org.apache.bookkeeper.client.api.WriteHandle in project pravega by pravega.

the class BookKeeperLog method initialize.

// endregion
// region DurableDataLog Implementation
/**
 * Open-Fences this BookKeeper log using the following protocol:
 * 1. Read Log Metadata from ZooKeeper.
 * 2. Fence at least the last 2 ledgers in the Ledger List.
 * 3. Create a new Ledger.
 * 3.1 If any of the steps so far fails, the process is interrupted at the point of failure, and no cleanup is attempted.
 * 4. Update Log Metadata using compare-and-set (this update contains the new ledger and new epoch).
 * 4.1 If CAS fails on metadata update, the newly created Ledger is deleted (this means we were fenced out by some
 * other instance) and no other update is performed.
 *
 * @param timeout Timeout for the operation.
 * @throws DataLogWriterNotPrimaryException If we were fenced-out during this process.
 * @throws DataLogNotAvailableException     If BookKeeper or ZooKeeper are not available.
 * @throws DataLogDisabledException         If the BookKeeperLog is disabled. No fencing is attempted in this case.
 * @throws DataLogInitializationException   If a general initialization error occurred.
 * @throws DurableDataLogException          If another type of exception occurred.
 */
@Override
public void initialize(Duration timeout) throws DurableDataLogException {
    List<Long> ledgersToDelete;
    LogMetadata newMetadata;
    synchronized (this.lock) {
        Preconditions.checkState(this.writeLedger == null, "BookKeeperLog is already initialized.");
        assert this.logMetadata == null : "writeLedger == null but logMetadata != null";
        // Get metadata about the current state of the log, if any.
        LogMetadata oldMetadata = loadMetadata();
        if (oldMetadata != null) {
            if (!oldMetadata.isEnabled()) {
                throw new DataLogDisabledException("BookKeeperLog is disabled. Cannot initialize.");
            }
            // Fence out ledgers.
            val emptyLedgerIds = Ledgers.fenceOut(oldMetadata.getLedgers(), this.bookKeeper, this.config, this.traceObjectId);
            // Update Metadata to reflect those newly found empty ledgers.
            oldMetadata = oldMetadata.updateLedgerStatus(emptyLedgerIds);
        }
        // Create new ledger.
        WriteHandle newLedger = Ledgers.create(this.bookKeeper, this.config, this.logId);
        log.info("{}: Created Ledger {}.", this.traceObjectId, newLedger.getId());
        // Update Metadata with new Ledger and persist to ZooKeeper.
        newMetadata = updateMetadata(oldMetadata, newLedger, true);
        LedgerMetadata ledgerMetadata = newMetadata.getLedger(newLedger.getId());
        assert ledgerMetadata != null : "cannot find newly added ledger metadata";
        this.writeLedger = new WriteLedger(newLedger, ledgerMetadata);
        this.logMetadata = newMetadata;
        ledgersToDelete = getLedgerIdsToDelete(oldMetadata, newMetadata);
    }
    // Delete the orphaned ledgers from BookKeeper.
    ledgersToDelete.forEach(id -> {
        try {
            Ledgers.delete(id, this.bookKeeper);
            log.info("{}: Deleted orphan empty ledger {}.", this.traceObjectId, id);
        } catch (DurableDataLogException ex) {
            // A failure here has no effect on the initialization of BookKeeperLog. In this case, the (empty) Ledger
            // will remain in BookKeeper until manually deleted by a cleanup tool.
            log.warn("{}: Unable to delete orphan empty ledger {}.", this.traceObjectId, id, ex);
        }
    });
    log.info("{}: Initialized (Epoch = {}, UpdateVersion = {}).", this.traceObjectId, newMetadata.getEpoch(), newMetadata.getUpdateVersion());
}

Example 10 with WriteHandle

use of org.apache.bookkeeper.client.api.WriteHandle in project pravega by pravega.

the class BookKeeperAdapter method createStream.

@Override
public CompletableFuture<Void> createStream(String logName, Duration timeout) {
    ensureRunning();
    return CompletableFuture.runAsync(() -> {
        WriteHandle ledger = null;
        boolean success = false;
        try {
            ledger = getBookKeeper().newCreateLedgerOp().withEnsembleSize(this.bkConfig.getBkEnsembleSize()).withWriteQuorumSize(this.bkConfig.getBkWriteQuorumSize()).withAckQuorumSize(this.bkConfig.getBkAckQuorumSize()).withDigestType(DigestType.CRC32C).withPassword(new byte[0]).execute().get();
            this.ledgers.put(logName, ledger);
            success = true;
        } catch (Exception ex) {
            throw new CompletionException(ex);
        } finally {
            if (!success) {
                this.ledgers.remove(logName);
                if (ledger != null) {
                    try {
                        ledger.close();
                    } catch (Exception ex) {
                        System.err.println(ex);
                    }
                }
            }
        }
    }, this.executor);
}