Examples with LedgerEntry org.apache.bookkeeper.client.LedgerEntry used on opensource projects

Example 16 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project bookkeeper by apache.

the class TestAuth method entryCount.

/**
 * check if the entry exists. Restart the bookie to allow
 * access
 */
private int entryCount(long ledgerId, ServerConfiguration bookieConf, ClientConfiguration clientConf) throws Exception {
    LOG.info("Counting entries in {}", ledgerId);
    for (ServerConfiguration conf : bsConfs) {
        conf.setBookieAuthProviderFactoryClass(AlwaysSucceedBookieAuthProviderFactory.class.getName());
    }
    clientConf.setClientAuthProviderFactoryClass(SendUntilCompleteClientAuthProviderFactory.class.getName());
    restartBookies();
    BookKeeper bkc = new BookKeeper(clientConf, zkc);
    LedgerHandle lh = bkc.openLedger(ledgerId, DigestType.CRC32, PASSWD);
    if (lh.getLastAddConfirmed() < 0) {
        return 0;
    }
    Enumeration<LedgerEntry> e = lh.readEntries(0, lh.getLastAddConfirmed());
    int count = 0;
    while (e.hasMoreElements()) {
        count++;
        assertTrue("Should match what we wrote", Arrays.equals(e.nextElement().getEntry(), ENTRY));
    }
    return count;
}

Example 17 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project bookkeeper by apache.

the class LedgerStorageCheckpointTest method testCheckPointForEntryLoggerWithMultipleActiveEntryLogs.

/*
     * This is complete end-to-end scenario.
     *
     * 1) This testcase uses MockInterleavedLedgerStorage, which extends
     * InterleavedLedgerStorage but doesn't do anything when Bookie is shutdown.
     * This is needed to simulate Bookie crash.
     * 2) entryLogPerLedger is enabled
     * 3) ledgers are created and entries are added.
     * 4) wait for flushInterval period for checkpoint to complete
     * 5) simulate bookie crash
     * 6) delete the journal files and lastmark file
     * 7) Now restart the Bookie
     * 8) validate that the entries which were written can be read successfully.
     */
@Test
public void testCheckPointForEntryLoggerWithMultipleActiveEntryLogs() throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setAutoRecoveryDaemonEnabled(false).setFlushInterval(3000).setBookiePort(PortManager.nextFreePort()).setEntryLogPerLedgerEnabled(true).setLedgerStorageClass(MockInterleavedLedgerStorage.class.getName());
    Assert.assertEquals("Number of JournalDirs", 1, conf.getJournalDirs().length);
    // we know there is only one ledgerDir
    File ledgerDir = Bookie.getCurrentDirectories(conf.getLedgerDirs())[0];
    BookieServer server = new BookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    final BookKeeper bkClient = new BookKeeper(clientConf);
    int numOfLedgers = 12;
    int numOfEntries = 100;
    byte[] dataBytes = "data".getBytes();
    AtomicBoolean receivedExceptionForAdd = new AtomicBoolean(false);
    LongStream.range(0, numOfLedgers).parallel().mapToObj((ledgerId) -> {
        LedgerHandle handle = null;
        try {
            handle = bkClient.createLedgerAdv(ledgerId, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
        } catch (BKException | InterruptedException exc) {
            receivedExceptionForAdd.compareAndSet(false, true);
            LOG.error("Got Exception while trying to create LedgerHandle for ledgerId: " + ledgerId, exc);
        }
        return handle;
    }).forEach((writeHandle) -> {
        IntStream.range(0, numOfEntries).forEach((entryId) -> {
            try {
                writeHandle.addEntry(entryId, dataBytes);
            } catch (BKException | InterruptedException exc) {
                receivedExceptionForAdd.compareAndSet(false, true);
                LOG.error("Got Exception while trying to AddEntry of ledgerId: " + writeHandle.getId() + " entryId: " + entryId, exc);
            }
        });
        try {
            writeHandle.close();
        } catch (BKException | InterruptedException e) {
            receivedExceptionForAdd.compareAndSet(false, true);
            LOG.error("Got Exception while trying to close writeHandle of ledgerId: " + writeHandle.getId(), e);
        }
    });
    Assert.assertFalse("There shouldn't be any exceptions while creating writeHandle and adding entries to writeHandle", receivedExceptionForAdd.get());
    executorController.advance(Duration.ofMillis(conf.getFlushInterval()));
    // since we have waited for more than flushInterval SyncThread should have checkpointed.
    // if entrylogperledger is not enabled, then we checkpoint only when currentLog in EntryLogger
    // is rotated. but if entrylogperledger is enabled, then we checkpoint for every flushInterval period
    File lastMarkFile = new File(ledgerDir, "lastMark");
    Assert.assertTrue("lastMark file must be existing, because checkpoint should have happened", lastMarkFile.exists());
    LogMark rolledLogMark = readLastMarkFile(lastMarkFile);
    Assert.assertNotEquals("rolledLogMark should not be zero, since checkpoint has happenend", 0, rolledLogMark.compare(new LogMark()));
    bkClient.close();
    // here we are calling shutdown, but MockInterleavedLedgerStorage shudown/flush
    // methods are noop, so entrylogger is not flushed as part of this shutdown
    // here we are trying to simulate Bookie crash, but there is no way to
    // simulate bookie abrupt crash
    server.shutdown();
    // delete journal files and lastMark, to make sure that we are not reading from
    // Journal file
    File[] journalDirs = conf.getJournalDirs();
    for (File journalDir : journalDirs) {
        File journalDirectory = Bookie.getCurrentDirectory(journalDir);
        List<Long> journalLogsId = Journal.listJournalIds(journalDirectory, null);
        for (long journalId : journalLogsId) {
            File journalFile = new File(journalDirectory, Long.toHexString(journalId) + ".txn");
            journalFile.delete();
        }
    }
    // we know there is only one ledgerDir
    lastMarkFile = new File(ledgerDir, "lastMark");
    lastMarkFile.delete();
    // now we are restarting BookieServer
    conf.setLedgerStorageClass(InterleavedLedgerStorage.class.getName());
    server = new BookieServer(conf);
    server.start();
    BookKeeper newBKClient = new BookKeeper(clientConf);
    // since Bookie checkpointed successfully before shutdown/crash,
    // we should be able to read from entryLogs though journal is deleted
    AtomicBoolean receivedExceptionForRead = new AtomicBoolean(false);
    LongStream.range(0, numOfLedgers).parallel().forEach((ledgerId) -> {
        try {
            LedgerHandle lh = newBKClient.openLedger(ledgerId, DigestType.CRC32, "passwd".getBytes());
            Enumeration<LedgerEntry> entries = lh.readEntries(0, numOfEntries - 1);
            while (entries.hasMoreElements()) {
                LedgerEntry entry = entries.nextElement();
                byte[] readData = entry.getEntry();
                Assert.assertEquals("Ledger Entry Data should match", new String("data".getBytes()), new String(readData));
            }
            lh.close();
        } catch (BKException | InterruptedException e) {
            receivedExceptionForRead.compareAndSet(false, true);
            LOG.error("Got Exception while trying to read entries of ledger, ledgerId: " + ledgerId, e);
        }
    });
    Assert.assertFalse("There shouldn't be any exceptions while creating readHandle and while reading" + "entries using readHandle", receivedExceptionForRead.get());
    newBKClient.close();
    server.shutdown();
}

Example 18 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project incubator-pulsar by apache.

the class SimpleBookKeeperTest method simpleTest.

@Test
public void simpleTest() throws Exception {
    LedgerHandle ledger = bkc.createLedger(DigestType.MAC, SECRET.getBytes());
    long ledgerId = ledger.getId();
    log.info("Writing to ledger: {}", ledgerId);
    for (int i = 0; i < 10; i++) {
        String content = "entry-" + i;
        ledger.addEntry(content.getBytes(Encoding));
    }
    ledger.close();
    ledger = bkc.openLedger(ledgerId, DigestType.MAC, SECRET.getBytes());
    Enumeration<LedgerEntry> entries = ledger.readEntries(0, 9);
    while (entries.hasMoreElements()) {
        LedgerEntry entry = entries.nextElement();
        String content = new String(entry.getEntry(), Encoding);
        log.info("Entry {}  lenght={} content='{}'", entry.getEntryId(), entry.getLength(), content);
    }
    ledger.close();
}

Example 19 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project distributedlog by twitter.

the class ReadUtils method asyncReadRecordFromEntries.

/**
 * Read record from a given range of ledger entries.
 *
 * @param streamName
 *          fully qualified stream name (used for logging)
 * @param ledgerDescriptor
 *          ledger descriptor.
 * @param handleCache
 *          ledger handle cache.
 * @param executorService
 *          executor service used for processing entries
 * @param context
 *          scan context
 * @return a future with the log record.
 */
private static Future<LogRecordWithDLSN> asyncReadRecordFromEntries(final String streamName, final LedgerDescriptor ledgerDescriptor, LedgerHandleCache handleCache, final LogSegmentMetadata metadata, final ExecutorService executorService, final ScanContext context, final LogRecordSelector selector) {
    final Promise<LogRecordWithDLSN> promise = new Promise<LogRecordWithDLSN>();
    final long startEntryId = context.curStartEntryId.get();
    final long endEntryId = context.curEndEntryId.get();
    if (LOG.isDebugEnabled()) {
        LOG.debug("{} reading entries [{} - {}] from {}.", new Object[] { streamName, startEntryId, endEntryId, ledgerDescriptor });
    }
    FutureEventListener<Enumeration<LedgerEntry>> readEntriesListener = new FutureEventListener<Enumeration<LedgerEntry>>() {

        @Override
        public void onSuccess(final Enumeration<LedgerEntry> entries) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("{} finished reading entries [{} - {}] from {}", new Object[] { streamName, startEntryId, endEntryId, ledgerDescriptor });
            }
            LogRecordWithDLSN record = null;
            while (entries.hasMoreElements()) {
                LedgerEntry entry = entries.nextElement();
                try {
                    visitEntryRecords(streamName, metadata, ledgerDescriptor.getLogSegmentSequenceNo(), entry, context, selector);
                } catch (IOException ioe) {
                    // exception is only thrown due to bad ledger entry, so it might be corrupted
                    // we shouldn't do anything beyond this point. throw the exception to application
                    promise.setException(ioe);
                    return;
                }
            }
            record = selector.result();
            if (LOG.isDebugEnabled()) {
                LOG.debug("{} got record from entries [{} - {}] of {} : {}", new Object[] { streamName, startEntryId, endEntryId, ledgerDescriptor, record });
            }
            promise.setValue(record);
        }

        @Override
        public void onFailure(final Throwable cause) {
            String errMsg = "Error reading entries [" + startEntryId + "-" + endEntryId + "] for reading record of " + streamName;
            promise.setException(new IOException(errMsg, BKException.create(FutureUtils.bkResultCode(cause))));
        }
    };
    handleCache.asyncReadEntries(ledgerDescriptor, startEntryId, endEntryId).addEventListener(FutureEventListenerRunnable.of(readEntriesListener, executorService));
    return promise;
}

Example 20 with LedgerEntry

use of org.apache.bookkeeper.client.LedgerEntry in project distributedlog by twitter.

the class LogSegmentReader method nextKeyValue.

@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    LogRecordWithDLSN record;
    currentRecord = null;
    if (null != reader) {
        record = reader.nextRecord();
        if (null != record) {
            currentRecord = record;
            readPos = record.getPositionWithinLogSegment();
            return true;
        } else {
            return false;
        }
    }
    ++entryId;
    if (entryId > lh.getLastAddConfirmed()) {
        return false;
    }
    try {
        Enumeration<LedgerEntry> entries = lh.readEntries(entryId, entryId);
        if (entries.hasMoreElements()) {
            LedgerEntry entry = entries.nextElement();
            reader = Entry.newBuilder().setLogSegmentInfo(metadata.getLogSegmentSequenceNumber(), metadata.getStartSequenceId()).setEntryId(entry.getEntryId()).setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(metadata.getVersion())).deserializeRecordSet(true).setInputStream(entry.getEntryInputStream()).buildReader();
        }
        return nextKeyValue();
    } catch (BKException e) {
        throw new IOException(e);
    }
}