Examples with BookieImpl org.apache.bookkeeper.bookie.BookieImpl used on opensource projects

Example 11 with BookieImpl

use of org.apache.bookkeeper.bookie.BookieImpl in project bookkeeper by apache.

the class ReadOnlyBookieTest method testBookieShouldTurnWritableFromReadOnly.

@Test
public void testBookieShouldTurnWritableFromReadOnly() throws Exception {
    killBookie(0);
    baseConf.setReadOnlyModeEnabled(true);
    startNewBookie();
    LedgerHandle ledger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    // Check new bookie with readonly mode enabled.
    File[] ledgerDirs = confByIndex(1).getLedgerDirs();
    assertEquals("Only one ledger dir should be present", 1, ledgerDirs.length);
    BookieImpl bookie = (BookieImpl) serverByIndex(1).getBookie();
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    File testDir = new File(ledgerDirs[0], "current");
    // Now add the current ledger dir to filled dirs list
    ledgerDirsManager.addToFilledDirs(testDir);
    try {
        ledger.addEntry("data".getBytes());
        fail("Should fail to add entry since there isn't enough bookies alive.");
    } catch (BKException.BKNotEnoughBookiesException e) {
    // Expected
    }
    bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(1))).get(30, TimeUnit.SECONDS);
    LOG.info("bookie is running {}, readonly {}.", bookie.isRunning(), bookie.isReadOnly());
    assertTrue("Bookie should be running and converted to readonly mode", bookie.isRunning() && bookie.isReadOnly());
    // should fail to create ledger
    try {
        bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
        fail("Should fail to create a ledger since there isn't enough bookies alive.");
    } catch (BKException.BKNotEnoughBookiesException bke) {
    // Expected.
    }
    // Now add the current ledger dir back to writable dirs list
    ledgerDirsManager.addToWritableDirs(testDir, true);
    bkc.waitForWritableBookie(BookieImpl.getBookieId(confByIndex(1))).get(30, TimeUnit.SECONDS);
    LOG.info("bookie is running {}, readonly {}.", bookie.isRunning(), bookie.isReadOnly());
    assertTrue("Bookie should be running and converted back to writable mode", bookie.isRunning() && !bookie.isReadOnly());
    LedgerHandle newLedger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    for (int i = 0; i < 10; i++) {
        newLedger.addEntry("data".getBytes());
    }
    Enumeration<LedgerEntry> readEntries = newLedger.readEntries(0, 9);
    while (readEntries.hasMoreElements()) {
        LedgerEntry entry = readEntries.nextElement();
        assertEquals("Entry should contain correct data", "data", new String(entry.getEntry()));
    }
}

Example 12 with BookieImpl

use of org.apache.bookkeeper.bookie.BookieImpl in project bookkeeper by apache.

the class ReadOnlyBookieTest method testReadFromReadOnlyBookieShouldBeSuccess.

/**
 * Try to read closed ledger from restarted ReadOnlyBookie.
 */
public void testReadFromReadOnlyBookieShouldBeSuccess() throws Exception {
    LedgerHandle ledger = bkc.createLedger(2, 2, DigestType.MAC, "".getBytes());
    for (int i = 0; i < 10; i++) {
        ledger.addEntry("data".getBytes());
    }
    ledger.close();
    confByIndex(1).setReadOnlyModeEnabled(true);
    confByIndex(1).setDiskCheckInterval(500);
    restartBookies();
    // Check new bookie with readonly mode enabled.
    File[] ledgerDirs = confByIndex(1).getLedgerDirs();
    assertEquals("Only one ledger dir should be present", 1, ledgerDirs.length);
    BookieImpl bookie = (BookieImpl) serverByIndex(1).getBookie();
    LedgerDirsManager ledgerDirsManager = bookie.getLedgerDirsManager();
    // Now add the current ledger dir to filled dirs list
    ledgerDirsManager.addToFilledDirs(new File(ledgerDirs[0], "current"));
    // Wait till Bookie converts to ReadOnly mode.
    Thread.sleep(1000);
    assertTrue("Bookie should be converted to readonly mode", bookie.isRunning() && bookie.isReadOnly());
    // Now kill the other bookie and read entries from the readonly bookie
    killBookie(0);
    Enumeration<LedgerEntry> readEntries = ledger.readEntries(0, 9);
    while (readEntries.hasMoreElements()) {
        LedgerEntry entry = readEntries.nextElement();
        assertEquals("Entry should contain correct data", "data", new String(entry.getEntry()));
    }
}

Example 13 with BookieImpl

use of org.apache.bookkeeper.bookie.BookieImpl in project bookkeeper by apache.

the class MdcContextTest method testAddFailsWithReadOnlyBookie.

@Test
public void testAddFailsWithReadOnlyBookie() throws Exception {
    for (int i = 0; i < 3; ++i) {
        Bookie bookie = serverByIndex(i).getBookie();
        File[] ledgerDirs = confByIndex(i).getLedgerDirs();
        LedgerDirsManager ledgerDirsManager = ((BookieImpl) bookie).getLedgerDirsManager();
        ledgerDirsManager.addToFilledDirs(new File(ledgerDirs[0], "current"));
    }
    ThreadContext.put(MDC_REQUEST_ID, "ledger_add_entry");
    try {
        lh.addEntry(0, entry);
        Assert.fail("should not get here");
    } catch (BKException bke) {
    // expected, pass
    }
    assertLogWithMdc("ledger_add_entry", "No writable ledger dirs below diskUsageThreshold");
    assertLogWithMdc("ledger_add_entry", "All ledger directories are non writable and no reserved space");
    assertLogWithMdc("ledger_add_entry", "Error writing entry:0 to ledger:0");
    assertLogWithMdc("ledger_add_entry", "Add for failed on bookie");
    assertLogWithMdc("ledger_add_entry", "Failed to find 1 bookies");
    assertLogWithMdc("ledger_add_entry", "Closing ledger 0 due to NotEnoughBookiesException");
}

Example 14 with BookieImpl

use of org.apache.bookkeeper.bookie.BookieImpl in project bookkeeper by apache.

the class Main method buildBookieServer.

/**
 * Build the bookie server.
 *
 * <p>The sequence of the components is:
 *
 * <pre>
 * - stats provider
 * - bookie server
 * - autorecovery daemon
 * - http service
 * </pre>
 *
 * @param conf bookie server configuration
 * @return lifecycle stack
 */
public static LifecycleComponentStack buildBookieServer(BookieConfiguration conf) throws Exception {
    final ComponentInfoPublisher componentInfoPublisher = new ComponentInfoPublisher();
    final Supplier<BookieServiceInfo> bookieServiceInfoProvider = () -> buildBookieServiceInfo(componentInfoPublisher);
    LifecycleComponentStack.Builder serverBuilder = LifecycleComponentStack.newBuilder().withComponentInfoPublisher(componentInfoPublisher).withName("bookie-server");
    // 1. build stats provider
    StatsProviderService statsProviderService = new StatsProviderService(conf);
    StatsLogger rootStatsLogger = statsProviderService.getStatsProvider().getStatsLogger("");
    serverBuilder.addComponent(statsProviderService);
    log.info("Load lifecycle component : {}", StatsProviderService.class.getName());
    // 2. Build metadata driver
    MetadataBookieDriver metadataDriver = BookieResources.createMetadataDriver(conf.getServerConf(), rootStatsLogger);
    serverBuilder.addComponent(new AutoCloseableLifecycleComponent("metadataDriver", metadataDriver));
    RegistrationManager rm = metadataDriver.createRegistrationManager();
    serverBuilder.addComponent(new AutoCloseableLifecycleComponent("registrationManager", rm));
    // 3. Build ledger manager
    LedgerManagerFactory lmFactory = metadataDriver.getLedgerManagerFactory();
    serverBuilder.addComponent(new AutoCloseableLifecycleComponent("lmFactory", lmFactory));
    LedgerManager ledgerManager = lmFactory.newLedgerManager();
    serverBuilder.addComponent(new AutoCloseableLifecycleComponent("ledgerManager", ledgerManager));
    // 4. Build bookie
    StatsLogger bookieStats = rootStatsLogger.scope(BOOKIE_SCOPE);
    DiskChecker diskChecker = BookieResources.createDiskChecker(conf.getServerConf());
    LedgerDirsManager ledgerDirsManager = BookieResources.createLedgerDirsManager(conf.getServerConf(), diskChecker, bookieStats.scope(LD_LEDGER_SCOPE));
    LedgerDirsManager indexDirsManager = BookieResources.createIndexDirsManager(conf.getServerConf(), diskChecker, bookieStats.scope(LD_INDEX_SCOPE), ledgerDirsManager);
    ByteBufAllocatorWithOomHandler allocator = BookieResources.createAllocator(conf.getServerConf());
    UncleanShutdownDetection uncleanShutdownDetection = new UncleanShutdownDetectionImpl(ledgerDirsManager);
    if (uncleanShutdownDetection.lastShutdownWasUnclean()) {
        log.info("Unclean shutdown detected. The bookie did not register a graceful shutdown prior to this boot.");
    }
    // bookie takes ownership of storage, so shuts it down
    LedgerStorage storage = null;
    DataIntegrityCheck integCheck = null;
    if (conf.getServerConf().isDataIntegrityCheckingEnabled()) {
        StatsLogger clientStats = bookieStats.scope(CLIENT_SCOPE);
        ClientConfiguration clientConfiguration = new ClientConfiguration(conf.getServerConf());
        clientConfiguration.setClientRole(ClientConfiguration.CLIENT_ROLE_SYSTEM);
        BookKeeper bkc = BookKeeper.forConfig(clientConfiguration).statsLogger(clientStats).build();
        serverBuilder.addComponent(new AutoCloseableLifecycleComponent("bkc", bkc));
        BookieId bookieId = BookieImpl.getBookieId(conf.getServerConf());
        ExecutorService rxExecutor = Executors.newFixedThreadPool(2, new ThreadFactoryBuilder().setNameFormat("rx-schedule-%d").setUncaughtExceptionHandler((t, ex) -> log.error("Uncaught exception on thread {}", t.getName(), ex)).build());
        Scheduler rxScheduler = Schedulers.from(rxExecutor);
        serverBuilder.addComponent(new RxSchedulerLifecycleComponent("rx-scheduler", conf, bookieStats, rxScheduler, rxExecutor));
        storage = BookieResources.createLedgerStorage(conf.getServerConf(), ledgerManager, ledgerDirsManager, indexDirsManager, bookieStats, allocator);
        EntryCopier copier = new EntryCopierImpl(bookieId, ((org.apache.bookkeeper.client.BookKeeper) bkc).getClientCtx().getBookieClient(), storage, Ticker.systemTicker());
        integCheck = new DataIntegrityCheckImpl(bookieId, ledgerManager, storage, copier, new BookKeeperAdmin(bkc, clientStats, clientConfiguration), rxScheduler);
        // run the preboot check to protect against data loss and to perform data repair
        if (!conf.getServerConf().getJournalWriteData() && uncleanShutdownDetection.lastShutdownWasUnclean()) {
            integCheck.runPreBootCheck("UNCLEAN_SHUTDOWN");
        }
        CookieValidation cookieValidation = new DataIntegrityCookieValidation(conf.getServerConf(), rm, integCheck);
        cookieValidation.checkCookies(storageDirectoriesFromConf(conf.getServerConf()));
    } else {
        CookieValidation cookieValidation = new LegacyCookieValidation(conf.getServerConf(), rm);
        cookieValidation.checkCookies(storageDirectoriesFromConf(conf.getServerConf()));
        storage = BookieResources.createLedgerStorage(conf.getServerConf(), ledgerManager, ledgerDirsManager, indexDirsManager, bookieStats, allocator);
    }
    Bookie bookie;
    if (conf.getServerConf().isForceReadOnlyBookie()) {
        bookie = new ReadOnlyBookie(conf.getServerConf(), rm, storage, diskChecker, ledgerDirsManager, indexDirsManager, bookieStats, allocator, bookieServiceInfoProvider);
    } else {
        bookie = new BookieImpl(conf.getServerConf(), rm, storage, diskChecker, ledgerDirsManager, indexDirsManager, bookieStats, allocator, bookieServiceInfoProvider);
    }
    // 5. build bookie server
    BookieService bookieService = new BookieService(conf, bookie, rootStatsLogger, allocator, uncleanShutdownDetection);
    serverBuilder.addComponent(bookieService);
    log.info("Load lifecycle component : {}", BookieService.class.getName());
    if (conf.getServerConf().isLocalScrubEnabled()) {
        serverBuilder.addComponent(new ScrubberService(rootStatsLogger.scope(ScrubberStats.SCOPE), conf, bookieService.getServer().getBookie().getLedgerStorage()));
    }
    // 6. build auto recovery
    if (conf.getServerConf().isAutoRecoveryDaemonEnabled()) {
        AutoRecoveryService autoRecoveryService = new AutoRecoveryService(conf, rootStatsLogger.scope(REPLICATION_SCOPE));
        serverBuilder.addComponent(autoRecoveryService);
        log.info("Load lifecycle component : {}", AutoRecoveryService.class.getName());
    }
    // 7. build data integrity check service
    if (conf.getServerConf().isDataIntegrityCheckingEnabled()) {
        checkNotNull(integCheck, "integCheck should have been initialized with the cookie validation");
        DataIntegrityService dataIntegrityService = new DataIntegrityService(conf, rootStatsLogger.scope(REPLICATION_SCOPE), integCheck);
        serverBuilder.addComponent(dataIntegrityService);
        log.info("Load lifecycle component : {}", DataIntegrityService.class.getName());
    }
    // 8. build http service
    if (conf.getServerConf().isHttpServerEnabled()) {
        BKHttpServiceProvider provider = new BKHttpServiceProvider.Builder().setBookieServer(bookieService.getServer()).setServerConfiguration(conf.getServerConf()).setStatsProvider(statsProviderService.getStatsProvider()).setLedgerManagerFactory(metadataDriver.getLedgerManagerFactory()).build();
        HttpService httpService = new HttpService(provider, conf, rootStatsLogger);
        serverBuilder.addComponent(httpService);
        log.info("Load lifecycle component : {}", HttpService.class.getName());
    }
    // 9. build extra services
    String[] extraComponents = conf.getServerConf().getExtraServerComponents();
    if (null != extraComponents) {
        try {
            List<ServerLifecycleComponent> components = loadServerComponents(extraComponents, conf, rootStatsLogger);
            for (ServerLifecycleComponent component : components) {
                serverBuilder.addComponent(component);
                log.info("Load lifecycle component : {}", component.getClass().getName());
            }
        } catch (Exception e) {
            if (conf.getServerConf().getIgnoreExtraServerComponentsStartupFailures()) {
                log.info("Failed to load extra components '{}' - {}. Continuing without those components.", StringUtils.join(extraComponents), e.getMessage());
            } else {
                throw e;
            }
        }
    }
    return serverBuilder.build();
}

Example 15 with BookieImpl

use of org.apache.bookkeeper.bookie.BookieImpl in project bookkeeper by apache.

the class LedgerCommandTest method setup.

public void setup() throws Exception {
    super.setup();
    mockServerConfigurationConstruction(serverconf -> {
        final ServerConfiguration defaultValue = new ServerConfiguration();
        when(serverconf.getLedgerStorageClass()).thenReturn(defaultValue.getLedgerStorageClass());
    });
    final MockedStatic<DbLedgerStorage> dbLedgerStorageMockedStatic = mockStatic(DbLedgerStorage.class);
    dbLedgerStorageMockedStatic.when(() -> DbLedgerStorage.readLedgerIndexEntries(anyLong(), any(ServerConfiguration.class), any(SingleDirectoryDbLedgerStorage.LedgerLoggerProcessor.class))).thenAnswer(invocation -> {
        SingleDirectoryDbLedgerStorage.LedgerLoggerProcessor p = invocation.getArgument(2);
        p.process(1L, 1L, 1L);
        return true;
    });
    LedgerCache.PageEntries e = mock(LedgerCache.PageEntries.class);
    LedgerCache.PageEntriesIterable i = mock(LedgerCache.PageEntriesIterable.class);
    metadata = mock(LedgerCache.LedgerIndexMetadata.class);
    mockConstruction(InterleavedLedgerStorage.class, (interleavedLedgerStorage, context) -> {
        when(interleavedLedgerStorage.getIndexEntries(anyLong())).thenReturn(i);
        when(interleavedLedgerStorage.readLedgerIndexMetadata(anyLong())).thenReturn(metadata);
    });
    final MockedStatic<BookieImpl> bookieMockedStatic = mockStatic(BookieImpl.class);
    bookieMockedStatic.when(() -> BookieImpl.mountLedgerStorageOffline(any(), any())).thenReturn(mock(LedgerStorage.class));
    when(i.iterator()).thenReturn(getPageIterator(e));
    LedgerEntryPage lep = mock(LedgerEntryPage.class);
    when(e.getLEP()).thenReturn(lep);
    when(metadata.getMasterKeyHex()).thenReturn("");
}