Examples with LedgerHandle org.apache.bookkeeper.client.LedgerHandle used on opensource projects

Example 96 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class LedgerStorageCheckpointTest method testCheckPointForEntryLoggerWithMultipleActiveEntryLogs.

/*
     * This is complete end-to-end scenario.
     *
     * 1) This testcase uses MockInterleavedLedgerStorage, which extends
     * InterleavedLedgerStorage but doesn't do anything when Bookie is shutdown.
     * This is needed to simulate Bookie crash.
     * 2) entryLogPerLedger is enabled
     * 3) ledgers are created and entries are added.
     * 4) wait for flushInterval period for checkpoint to complete
     * 5) simulate bookie crash
     * 6) delete the journal files and lastmark file
     * 7) Now restart the Bookie
     * 8) validate that the entries which were written can be read successfully.
     */
@Test
public void testCheckPointForEntryLoggerWithMultipleActiveEntryLogs() throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setAutoRecoveryDaemonEnabled(false).setFlushInterval(3000).setBookiePort(PortManager.nextFreePort()).setEntryLogPerLedgerEnabled(true).setLedgerStorageClass(MockInterleavedLedgerStorage.class.getName());
    Assert.assertEquals("Number of JournalDirs", 1, conf.getJournalDirs().length);
    // we know there is only one ledgerDir
    File ledgerDir = Bookie.getCurrentDirectories(conf.getLedgerDirs())[0];
    BookieServer server = new BookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    final BookKeeper bkClient = new BookKeeper(clientConf);
    int numOfLedgers = 12;
    int numOfEntries = 100;
    byte[] dataBytes = "data".getBytes();
    AtomicBoolean receivedExceptionForAdd = new AtomicBoolean(false);
    LongStream.range(0, numOfLedgers).parallel().mapToObj((ledgerId) -> {
        LedgerHandle handle = null;
        try {
            handle = bkClient.createLedgerAdv(ledgerId, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
        } catch (BKException | InterruptedException exc) {
            receivedExceptionForAdd.compareAndSet(false, true);
            LOG.error("Got Exception while trying to create LedgerHandle for ledgerId: " + ledgerId, exc);
        }
        return handle;
    }).forEach((writeHandle) -> {
        IntStream.range(0, numOfEntries).forEach((entryId) -> {
            try {
                writeHandle.addEntry(entryId, dataBytes);
            } catch (BKException | InterruptedException exc) {
                receivedExceptionForAdd.compareAndSet(false, true);
                LOG.error("Got Exception while trying to AddEntry of ledgerId: " + writeHandle.getId() + " entryId: " + entryId, exc);
            }
        });
        try {
            writeHandle.close();
        } catch (BKException | InterruptedException e) {
            receivedExceptionForAdd.compareAndSet(false, true);
            LOG.error("Got Exception while trying to close writeHandle of ledgerId: " + writeHandle.getId(), e);
        }
    });
    Assert.assertFalse("There shouldn't be any exceptions while creating writeHandle and adding entries to writeHandle", receivedExceptionForAdd.get());
    executorController.advance(Duration.ofMillis(conf.getFlushInterval()));
    // since we have waited for more than flushInterval SyncThread should have checkpointed.
    // if entrylogperledger is not enabled, then we checkpoint only when currentLog in EntryLogger
    // is rotated. but if entrylogperledger is enabled, then we checkpoint for every flushInterval period
    File lastMarkFile = new File(ledgerDir, "lastMark");
    Assert.assertTrue("lastMark file must be existing, because checkpoint should have happened", lastMarkFile.exists());
    LogMark rolledLogMark = readLastMarkFile(lastMarkFile);
    Assert.assertNotEquals("rolledLogMark should not be zero, since checkpoint has happenend", 0, rolledLogMark.compare(new LogMark()));
    bkClient.close();
    // here we are calling shutdown, but MockInterleavedLedgerStorage shudown/flush
    // methods are noop, so entrylogger is not flushed as part of this shutdown
    // here we are trying to simulate Bookie crash, but there is no way to
    // simulate bookie abrupt crash
    server.shutdown();
    // delete journal files and lastMark, to make sure that we are not reading from
    // Journal file
    File[] journalDirs = conf.getJournalDirs();
    for (File journalDir : journalDirs) {
        File journalDirectory = Bookie.getCurrentDirectory(journalDir);
        List<Long> journalLogsId = Journal.listJournalIds(journalDirectory, null);
        for (long journalId : journalLogsId) {
            File journalFile = new File(journalDirectory, Long.toHexString(journalId) + ".txn");
            journalFile.delete();
        }
    }
    // we know there is only one ledgerDir
    lastMarkFile = new File(ledgerDir, "lastMark");
    lastMarkFile.delete();
    // now we are restarting BookieServer
    conf.setLedgerStorageClass(InterleavedLedgerStorage.class.getName());
    server = new BookieServer(conf);
    server.start();
    BookKeeper newBKClient = new BookKeeper(clientConf);
    // since Bookie checkpointed successfully before shutdown/crash,
    // we should be able to read from entryLogs though journal is deleted
    AtomicBoolean receivedExceptionForRead = new AtomicBoolean(false);
    LongStream.range(0, numOfLedgers).parallel().forEach((ledgerId) -> {
        try {
            LedgerHandle lh = newBKClient.openLedger(ledgerId, DigestType.CRC32, "passwd".getBytes());
            Enumeration<LedgerEntry> entries = lh.readEntries(0, numOfEntries - 1);
            while (entries.hasMoreElements()) {
                LedgerEntry entry = entries.nextElement();
                byte[] readData = entry.getEntry();
                Assert.assertEquals("Ledger Entry Data should match", new String("data".getBytes()), new String(readData));
            }
            lh.close();
        } catch (BKException | InterruptedException e) {
            receivedExceptionForRead.compareAndSet(false, true);
            LOG.error("Got Exception while trying to read entries of ledger, ledgerId: " + ledgerId, e);
        }
    });
    Assert.assertFalse("There shouldn't be any exceptions while creating readHandle and while reading" + "entries using readHandle", receivedExceptionForRead.get());
    newBKClient.close();
    server.shutdown();
}

Example 97 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class TestGcOverreplicatedLedger method testGcOverreplicatedLedger.

@Test
public void testGcOverreplicatedLedger() throws Exception {
    LedgerHandle lh = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    activeLedgers.put(lh.getId(), true);
    final AtomicReference<LedgerMetadata> newLedgerMetadata = new AtomicReference<>(null);
    final CountDownLatch latch = new CountDownLatch(1);
    ledgerManager.readLedgerMetadata(lh.getId(), new GenericCallback<LedgerMetadata>() {

        @Override
        public void operationComplete(int rc, LedgerMetadata result) {
            if (rc == BKException.Code.OK) {
                newLedgerMetadata.set(result);
            }
            latch.countDown();
        }
    });
    latch.await();
    if (newLedgerMetadata.get() == null) {
        Assert.fail("No ledger metadata found");
    }
    BookieSocketAddress bookieNotInEnsemble = getBookieNotInEnsemble(newLedgerMetadata.get());
    ServerConfiguration bkConf = getBkConf(bookieNotInEnsemble);
    bkConf.setGcOverreplicatedLedgerWaitTime(10, TimeUnit.MILLISECONDS);
    lh.close();
    final CompactableLedgerStorage mockLedgerStorage = new MockLedgerStorage();
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(ledgerManager, mockLedgerStorage, bkConf, NullStatsLogger.INSTANCE);
    Thread.sleep(bkConf.getGcOverreplicatedLedgerWaitTimeMillis() + 1);
    garbageCollector.gc(new GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            try {
                mockLedgerStorage.deleteLedger(ledgerId);
            } catch (IOException e) {
                e.printStackTrace();
                return;
            }
        }
    });
    Assert.assertFalse(activeLedgers.containsKey(lh.getId()));
}

Example 98 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class TestGcOverreplicatedLedger method testNoGcOfLedger.

@Test
public void testNoGcOfLedger() throws Exception {
    LedgerHandle lh = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    activeLedgers.put(lh.getId(), true);
    final AtomicReference<LedgerMetadata> newLedgerMetadata = new AtomicReference<>(null);
    final CountDownLatch latch = new CountDownLatch(1);
    ledgerManager.readLedgerMetadata(lh.getId(), new GenericCallback<LedgerMetadata>() {

        @Override
        public void operationComplete(int rc, LedgerMetadata result) {
            if (rc == BKException.Code.OK) {
                newLedgerMetadata.set(result);
            }
            latch.countDown();
        }
    });
    latch.await();
    if (newLedgerMetadata.get() == null) {
        Assert.fail("No ledger metadata found");
    }
    BookieSocketAddress address = null;
    SortedMap<Long, ArrayList<BookieSocketAddress>> ensembleMap = newLedgerMetadata.get().getEnsembles();
    for (ArrayList<BookieSocketAddress> ensemble : ensembleMap.values()) {
        address = ensemble.get(0);
    }
    ServerConfiguration bkConf = getBkConf(address);
    bkConf.setGcOverreplicatedLedgerWaitTime(10, TimeUnit.MILLISECONDS);
    lh.close();
    final CompactableLedgerStorage mockLedgerStorage = new MockLedgerStorage();
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(ledgerManager, mockLedgerStorage, bkConf, NullStatsLogger.INSTANCE);
    Thread.sleep(bkConf.getGcOverreplicatedLedgerWaitTimeMillis() + 1);
    garbageCollector.gc(new GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            try {
                mockLedgerStorage.deleteLedger(ledgerId);
            } catch (IOException e) {
                e.printStackTrace();
                return;
            }
        }
    });
    Assert.assertTrue(activeLedgers.containsKey(lh.getId()));
}

Example 99 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class DbLedgerStorageBookieTest method testRecoveryEmptyLedger.

@Test
public void testRecoveryEmptyLedger() throws Exception {
    LedgerHandle lh1 = bkc.createLedger(1, 1, DigestType.MAC, new byte[0]);
    // Force ledger close & recovery
    LedgerHandle lh2 = bkc.openLedger(lh1.getId(), DigestType.MAC, new byte[0]);
    assertEquals(0, lh2.getLength());
    assertEquals(-1, lh2.getLastAddConfirmed());
}

Example 100 with LedgerHandle

use of org.apache.bookkeeper.client.LedgerHandle in project bookkeeper by apache.

the class DbLedgerStorageBookieTest method testV2ReadWrite.

@Test
public void testV2ReadWrite() throws Exception {
    ClientConfiguration conf = new ClientConfiguration();
    conf.setUseV2WireProtocol(true);
    conf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkc = new BookKeeper(conf);
    LedgerHandle lh1 = bkc.createLedger(1, 1, DigestType.CRC32, new byte[0]);
    lh1.addEntry("Foobar".getBytes());
    lh1.close();
    LedgerHandle lh2 = bkc.openLedger(lh1.getId(), DigestType.CRC32, new byte[0]);
    assertEquals(0, lh2.getLastAddConfirmed());
    assertEquals(new String(lh2.readEntries(0, 0).nextElement().getEntry()), "Foobar");
}