Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 56 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class LedgerStorageCheckpointTest method testCheckpointOfSLSWhenEntryLogIsRotated.

public void testCheckpointOfSLSWhenEntryLogIsRotated(boolean entryLogPerLedgerEnabled) throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setAutoRecoveryDaemonEnabled(false).setFlushInterval(30000).setBookiePort(PortManager.nextFreePort()).setEntryLogPerLedgerEnabled(entryLogPerLedgerEnabled).setLedgerStorageClass(SortedLedgerStorage.class.getName()).setSkipListSizeLimit(1 * 1000 * 1000).setEntryLogSizeLimit(2 * 1000 * 1000);
    Assert.assertEquals("Number of JournalDirs", 1, conf.getJournalDirs().length);
    // we know there is only one ledgerDir
    File ledgerDir = Bookie.getCurrentDirectories(conf.getLedgerDirs())[0];
    BookieServer server = new BookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkClient = new BookKeeper(clientConf);
    InterleavedLedgerStorage ledgerStorage = (InterleavedLedgerStorage) server.getBookie().ledgerStorage;
    Random rand = new Random();
    byte[] dataBytes = new byte[10 * 1000];
    rand.nextBytes(dataBytes);
    int numOfEntries = ((int) conf.getEntryLogSizeLimit() + (100 * 1000)) / dataBytes.length;
    LedgerHandle handle = bkClient.createLedgerAdv(10, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
    for (int j = 0; j < numOfEntries; j++) {
        handle.addEntry(j, dataBytes);
    }
    handle.close();
    // sleep for a bit for checkpoint to do its task
    executorController.advance(Duration.ofMillis(500));
    File lastMarkFile = new File(ledgerDir, "lastMark");
    LogMark rolledLogMark = readLastMarkFile(lastMarkFile);
    if (entryLogPerLedgerEnabled) {
        Assert.assertEquals("rolledLogMark should be zero, since checkpoint" + "shouldn't have happened when entryLog is rotated", 0, rolledLogMark.compare(new LogMark()));
    } else {
        Assert.assertNotEquals("rolledLogMark shouldn't be zero, since checkpoint" + "should have happened when entryLog is rotated", 0, rolledLogMark.compare(new LogMark()));
    }
    bkClient.close();
    server.shutdown();
}

Example 57 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class LedgerStorageCheckpointTest method testPeriodicCheckpointForLedgerStorage.

public void testPeriodicCheckpointForLedgerStorage(String ledgerStorageClassName) throws Exception {
    File tmpDir = createTempDir("DiskCheck", "test");
    final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration().setZkServers(zkUtil.getZooKeeperConnectString()).setZkTimeout(5000).setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new String[] { tmpDir.getPath() }).setAutoRecoveryDaemonEnabled(false).setFlushInterval(2000).setBookiePort(PortManager.nextFreePort()).setEntryLogPerLedgerEnabled(true).setLedgerStorageClass(ledgerStorageClassName);
    Assert.assertEquals("Number of JournalDirs", 1, conf.getJournalDirs().length);
    // we know there is only one ledgerDir
    File ledgerDir = Bookie.getCurrentDirectories(conf.getLedgerDirs())[0];
    BookieServer server = new BookieServer(conf);
    server.start();
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper bkClient = new BookKeeper(clientConf);
    int numOfLedgers = 2;
    int numOfEntries = 5;
    byte[] dataBytes = "data".getBytes();
    for (int i = 0; i < numOfLedgers; i++) {
        int ledgerIndex = i;
        LedgerHandle handle = bkClient.createLedgerAdv((long) i, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
        for (int j = 0; j < numOfEntries; j++) {
            handle.addEntry(j, dataBytes);
        }
        handle.close();
    }
    LastLogMark lastLogMarkAfterFirstSetOfAdds = server.getBookie().journals.get(0).getLastLogMark();
    LogMark curMarkAfterFirstSetOfAdds = lastLogMarkAfterFirstSetOfAdds.getCurMark();
    File lastMarkFile = new File(ledgerDir, "lastMark");
    // lastMark file should be zero, because checkpoint hasn't happenend
    LogMark logMarkFileBeforeCheckpoint = readLastMarkFile(lastMarkFile);
    Assert.assertEquals("lastMarkFile before checkpoint should be zero", 0, logMarkFileBeforeCheckpoint.compare(new LogMark()));
    // wait for flushInterval for SyncThread to do next iteration of checkpoint
    executorController.advance(Duration.ofMillis(conf.getFlushInterval()));
    /*
         * since we have waited for more than flushInterval SyncThread should
         * have checkpointed. if entrylogperledger is not enabled, then we
         * checkpoint only when currentLog in EntryLogger is rotated. but if
         * entrylogperledger is enabled, then we checkpoint for every
         * flushInterval period
         */
    Assert.assertTrue("lastMark file must be existing, because checkpoint should have happened", lastMarkFile.exists());
    LastLogMark lastLogMarkAfterCheckpoint = server.getBookie().journals.get(0).getLastLogMark();
    LogMark curMarkAfterCheckpoint = lastLogMarkAfterCheckpoint.getCurMark();
    LogMark rolledLogMark = readLastMarkFile(lastMarkFile);
    Assert.assertNotEquals("rolledLogMark should not be zero, since checkpoint has happenend", 0, rolledLogMark.compare(new LogMark()));
    /*
         * Curmark should be equal before and after checkpoint, because we didnt
         * add new entries during this period
         */
    Assert.assertTrue("Curmark should be equal before and after checkpoint", curMarkAfterCheckpoint.compare(curMarkAfterFirstSetOfAdds) == 0);
    /*
         * Curmark after checkpoint should be equal to rolled logmark, because
         * we checkpointed
         */
    Assert.assertTrue("Curmark after first set of adds should be equal to rolled logmark", curMarkAfterCheckpoint.compare(rolledLogMark) == 0);
    // add more ledger/entries
    for (int i = numOfLedgers; i < 2 * numOfLedgers; i++) {
        int ledgerIndex = i;
        LedgerHandle handle = bkClient.createLedgerAdv((long) i, 1, 1, 1, DigestType.CRC32, "passwd".getBytes(), null);
        for (int j = 0; j < numOfEntries; j++) {
            handle.addEntry(j, dataBytes);
        }
        handle.close();
    }
    // wait for flushInterval for SyncThread to do next iteration of checkpoint
    executorController.advance(Duration.ofMillis(conf.getFlushInterval()));
    LastLogMark lastLogMarkAfterSecondSetOfAdds = server.getBookie().journals.get(0).getLastLogMark();
    LogMark curMarkAfterSecondSetOfAdds = lastLogMarkAfterSecondSetOfAdds.getCurMark();
    rolledLogMark = readLastMarkFile(lastMarkFile);
    /*
         * Curmark after checkpoint should be equal to rolled logmark, because
         * we checkpointed
         */
    Assert.assertTrue("Curmark after second set of adds should be equal to rolled logmark", curMarkAfterSecondSetOfAdds.compare(rolledLogMark) == 0);
    server.shutdown();
    bkClient.close();
}

Example 58 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class GSSAPIBookKeeperTest method entryCount.

/**
 * check if the entry exists. Restart the bookie to allow access
 */
private int entryCount(long ledgerId, ClientConfiguration clientConf) throws Exception {
    LOG.info("Counting entries in {}", ledgerId);
    for (ServerConfiguration conf : bsConfs) {
        conf.setUseHostNameAsBookieID(true);
        conf.setBookieAuthProviderFactoryClass(SASLBookieAuthProviderFactory.class.getName());
    }
    clientConf.setClientAuthProviderFactoryClass(SASLClientProviderFactory.class.getName());
    restartBookies();
    try (BookKeeper bkc = new BookKeeper(clientConf, zkc);
        LedgerHandle lh = bkc.openLedger(ledgerId, DigestType.CRC32, PASSWD)) {
        if (lh.getLastAddConfirmed() < 0) {
            return 0;
        }
        Enumeration<LedgerEntry> e = lh.readEntries(0, lh.getLastAddConfirmed());
        int count = 0;
        while (e.hasMoreElements()) {
            count++;
            assertTrue("Should match what we wrote", Arrays.equals(e.nextElement().getEntry(), ENTRY));
        }
        return count;
    }
}

Example 59 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class GSSAPIBookKeeperTest method connectAndWriteToBookie.

// we pass in ledgerId because the method may throw exceptions
private void connectAndWriteToBookie(ClientConfiguration conf, AtomicLong ledgerWritten) throws BKException, InterruptedException, IOException, KeeperException {
    LOG.info("Connecting to bookie");
    try (BookKeeper bkc = new BookKeeper(conf, zkc)) {
        LedgerHandle l = bkc.createLedger(1, 1, DigestType.CRC32, PASSWD);
        ledgerWritten.set(l.getId());
        l.addEntry(ENTRY);
        l.close();
    }
}

Example 60 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project bookkeeper by apache.

the class TestHttpService method testGetLedgerMetaService.

@Test
public void testGetLedgerMetaService() throws Exception {
    baseConf.setZkServers(zkUtil.getZooKeeperConnectString());
    BookKeeper.DigestType digestType = BookKeeper.DigestType.CRC32;
    int numLedgers = 4;
    int numMsgs = 100;
    LedgerHandle[] lh = new LedgerHandle[numLedgers];
    // create ledgers
    for (int i = 0; i < numLedgers; i++) {
        lh[i] = bkc.createLedger(digestType, "password".getBytes());
    }
    String content = "Apache BookKeeper is cool!";
    // add entries
    for (int i = 0; i < numMsgs; i++) {
        for (int j = 0; j < numLedgers; j++) {
            lh[j].addEntry(content.getBytes());
        }
    }
    // close ledgers
    for (int i = 0; i < numLedgers; i++) {
        lh[i].close();
    }
    HttpEndpointService getLedgerMetaService = bkHttpServiceProvider.provideHttpEndpointService(HttpServer.ApiType.GET_LEDGER_META);
    // 1,  null parameters of GET, should return NOT_FOUND
    HttpServiceRequest request1 = new HttpServiceRequest(null, HttpServer.Method.GET, null);
    HttpServiceResponse response1 = getLedgerMetaService.handle(request1);
    assertEquals(HttpServer.StatusCode.NOT_FOUND.getValue(), response1.getStatusCode());
    // 2,  parameters for GET first ledger, should return OK, and contains metadata
    HashMap<String, String> params = Maps.newHashMap();
    Long ledgerId = Long.valueOf(lh[0].getId());
    params.put("ledger_id", ledgerId.toString());
    HttpServiceRequest request2 = new HttpServiceRequest(null, HttpServer.Method.GET, params);
    HttpServiceResponse response2 = getLedgerMetaService.handle(request2);
    assertEquals(HttpServer.StatusCode.OK.getValue(), response2.getStatusCode());
    @SuppressWarnings("unchecked") HashMap<String, String> respBody = JsonUtil.fromJson(response2.getBody(), HashMap.class);
    assertEquals(1, respBody.size());
    // verify LedgerMetadata content is equal
    assertTrue(respBody.get(ledgerId.toString()).toString().equals(new String(lh[0].getLedgerMetadata().serialize())));
}