Examples with LedgerHandle org.apache.bookkeeper.client.LedgerHandle used on opensource projects

Example 16 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.

the class TestLedgerAllocatorPool method testRescueAllocators.

@Test(timeout = 60000)
public void testRescueAllocators() throws Exception {
    String allocationPath = "/rescueAllocators";
    int numAllocators = 3;
    LedgerAllocatorPool pool = new LedgerAllocatorPool(allocationPath, numAllocators, dlConf, zkc, bkc, allocationExecutor);
    List<ZKTransaction> pendingTxns = Lists.newArrayListWithExpectedSize(numAllocators);
    List<String> allocatePaths = Lists.newArrayListWithExpectedSize(numAllocators);
    for (int i = 0; i < numAllocators; i++) {
        ZKTransaction txn = newTxn();
        pool.allocate();
        LedgerHandle lh = FutureUtils.result(pool.tryObtain(txn, NULL_LISTENER));
        // get the corresponding ledger allocator
        SimpleLedgerAllocator sla = pool.getLedgerAllocator(lh);
        String slaPath = sla.allocatePath;
        logger.info("Allocated ledger {} from path {}", lh.getId(), slaPath);
        pendingTxns.add(txn);
        allocatePaths.add(slaPath);
    }
    for (int i = 0; i < numAllocators; i++) {
        ZKTransaction txn = pendingTxns.get(i);
        String slaPath = allocatePaths.get(i);
        // execute the transaction to confirm/abort obtain
        FutureUtils.result(txn.execute());
        // introduce error to individual ledger allocator
        byte[] data = zkc.get().getData(slaPath, false, new Stat());
        zkc.get().setData(slaPath, data, -1);
    }
    int numSuccess = 0;
    Set<String> allocatedPathSet = new HashSet<String>();
    while (numSuccess < 2 * numAllocators) {
        try {
            pool.allocate();
            ZKTransaction txn = newTxn();
            LedgerHandle lh = FutureUtils.result(pool.tryObtain(txn, NULL_LISTENER));
            // get the corresponding ledger allocator
            SimpleLedgerAllocator sla = pool.getLedgerAllocator(lh);
            String slaPath = sla.allocatePath;
            logger.info("Allocated ledger {} from path {}", lh.getId(), slaPath);
            allocatedPathSet.add(slaPath);
            FutureUtils.result(txn.execute());
            ++numSuccess;
        } catch (IOException ioe) {
        // continue
        }
    }
    assertEquals(2 * numAllocators, numSuccess);
    assertEquals(numAllocators, allocatedPathSet.size());
    Utils.close(pool);
}

Example 17 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.

the class DLAuditor method calculateStreamSpaceUsage.

private long calculateStreamSpaceUsage(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String stream) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
    long totalBytes = 0;
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            try {
                LedgerHandle lh = getBookKeeperClient(factory).get().openLedgerNoRecovery(segment.getLedgerId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
                totalBytes += lh.getLength();
                lh.close();
            } catch (BKException e) {
                logger.error("Failed to open ledger {} : ", segment.getLedgerId(), e);
                throw new IOException("Failed to open ledger " + segment.getLedgerId(), e);
            } catch (InterruptedException e) {
                logger.warn("Interrupted on opening ledger {} : ", segment.getLedgerId(), e);
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLedgerId(), e);
            }
        }
    } finally {
        dlm.close();
    }
    return totalBytes;
}

Example 18 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.

the class LedgerAllocatorPool method tryObtain.

@Override
public Future<LedgerHandle> tryObtain(final Transaction<Object> txn, final Transaction.OpListener<LedgerHandle> listener) {
    final SimpleLedgerAllocator allocator;
    synchronized (this) {
        if (allocatingList.isEmpty()) {
            return Future.exception(new IOException("No ledger allocator available under " + poolPath + "."));
        } else {
            allocator = allocatingList.removeFirst();
        }
    }
    final Promise<LedgerHandle> tryObtainPromise = new Promise<LedgerHandle>();
    final FutureEventListener<LedgerHandle> tryObtainListener = new FutureEventListener<LedgerHandle>() {

        @Override
        public void onSuccess(LedgerHandle lh) {
            synchronized (LedgerAllocatorPool.this) {
                obtainMap.put(lh, allocator);
                reverseObtainMap.put(allocator, lh);
                tryObtainPromise.setValue(lh);
            }
        }

        @Override
        public void onFailure(Throwable cause) {
            try {
                rescueAllocator(allocator);
            } catch (IOException ioe) {
                logger.info("Failed to rescue allocator {}", allocator.allocatePath, ioe);
            }
            tryObtainPromise.setException(cause);
        }
    };
    allocator.tryObtain(txn, new Transaction.OpListener<LedgerHandle>() {

        @Override
        public void onCommit(LedgerHandle lh) {
            confirmObtain(allocator);
            listener.onCommit(lh);
        }

        @Override
        public void onAbort(Throwable t) {
            abortObtain(allocator);
            listener.onAbort(t);
        }
    }).addEventListener(tryObtainListener);
    return tryObtainPromise;
}

Example 19 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.

the class TestRollLogSegments method testCaughtUpReaderOnLogSegmentRolling.

@FlakyTest
@Test(timeout = 60000)
public void testCaughtUpReaderOnLogSegmentRolling() throws Exception {
    String name = "distrlog-caughtup-reader-on-logsegment-rolling";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setOutputBufferSize(4 * 1024 * 1024);
    confLocal.setTraceReadAheadMetadataChanges(true);
    confLocal.setEnsembleSize(1);
    confLocal.setWriteQuorumSize(1);
    confLocal.setAckQuorumSize(1);
    confLocal.setReadLACLongPollTimeout(99999999);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    // 1) writer added 5 entries.
    final int numEntries = 5;
    for (int i = 1; i <= numEntries; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(i));
        writer.setReadyToFlush();
        writer.flushAndSync();
    }
    BKDistributedLogManager readDLM = (BKDistributedLogManager) createNewDLM(confLocal, name);
    final BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // 2) reader should be able to read 5 entries.
    for (long i = 1; i <= numEntries; i++) {
        LogRecordWithDLSN record = Await.result(reader.readNext());
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(i, record.getTransactionId());
        assertEquals(record.getTransactionId() - 1, record.getSequenceId());
    }
    BKLogSegmentWriter perStreamWriter = writer.segmentWriter;
    BookKeeperClient bkc = readDLM.getReaderBKC();
    LedgerHandle readLh = bkc.get().openLedgerNoRecovery(getLedgerHandle(perStreamWriter).getId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    // Writer moved to lac = 9, while reader knows lac = 8 and moving to wait on 9
    checkAndWaitWriterReaderPosition(perStreamWriter, 9, reader, 9, readLh, 8);
    // write 6th record
    writer.write(DLMTestUtil.getLogRecordInstance(numEntries + 1));
    writer.setReadyToFlush();
    // Writer moved to lac = 10, while reader knows lac = 9 and moving to wait on 10
    checkAndWaitWriterReaderPosition(perStreamWriter, 10, reader, 10, readLh, 9);
    // write records without commit to simulate similar failure cases
    writer.write(DLMTestUtil.getLogRecordInstance(numEntries + 2));
    writer.setReadyToFlush();
    // Writer moved to lac = 11, while reader knows lac = 10 and moving to wait on 11
    checkAndWaitWriterReaderPosition(perStreamWriter, 11, reader, 11, readLh, 10);
    while (null == reader.bkLedgerManager.readAheadWorker.getMetadataNotification()) {
        Thread.sleep(1000);
    }
    logger.info("Waiting for long poll getting interrupted with metadata changed");
    // simulate a recovery without closing ledger causing recording wrong last dlsn
    BKLogWriteHandler writeHandler = writer.getCachedWriteHandler();
    writeHandler.completeAndCloseLogSegment(writeHandler.inprogressZNodeName(perStreamWriter.getLogSegmentId(), perStreamWriter.getStartTxId(), perStreamWriter.getLogSegmentSequenceNumber()), perStreamWriter.getLogSegmentSequenceNumber(), perStreamWriter.getLogSegmentId(), perStreamWriter.getStartTxId(), perStreamWriter.getLastTxId(), perStreamWriter.getPositionWithinLogSegment() - 1, 9, 0);
    BKSyncLogWriter anotherWriter = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    anotherWriter.write(DLMTestUtil.getLogRecordInstance(numEntries + 3));
    anotherWriter.setReadyToFlush();
    anotherWriter.flushAndSync();
    anotherWriter.closeAndComplete();
    for (long i = numEntries + 1; i <= numEntries + 3; i++) {
        LogRecordWithDLSN record = Await.result(reader.readNext());
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(i, record.getTransactionId());
    }
    Utils.close(reader);
    readDLM.close();
}

Example 20 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project distributedlog by twitter.

the class TestLedgerAllocator method testCloseAllocatorAfterConfirm.

/**
     * {@link https://issues.apache.org/jira/browse/DL-26}
     */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testCloseAllocatorAfterConfirm() throws Exception {
    String allocationPath = "/allocation2";
    SimpleLedgerAllocator allocator = createAllocator(allocationPath);
    allocator.allocate();
    ZKTransaction txn = newTxn();
    // close during obtaining ledger.
    LedgerHandle lh = FutureUtils.result(allocator.tryObtain(txn, NULL_LISTENER));
    FutureUtils.result(txn.execute());
    Utils.close(allocator);
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals(0, data.length);
    // the ledger is not deleted.
    bkc.get().openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8));
}