Examples with BookKeeperAdmin org.apache.bookkeeper.client.BookKeeperAdmin used on opensource projects

Example 6 with BookKeeperAdmin

use of org.apache.bookkeeper.client.BookKeeperAdmin in project bookkeeper by apache.

the class Auditor method initialize.

private void initialize(ServerConfiguration conf, ZooKeeper zkc) throws UnavailableException {
    try {
        ClientConfiguration clientConfiguration = new ClientConfiguration(conf);
        clientConfiguration.setClientRole(ClientConfiguration.CLIENT_ROLE_SYSTEM);
        LOG.info("AuthProvider used by the Auditor is {}", clientConfiguration.getClientAuthProviderFactoryClass());
        this.bkc = new BookKeeper(clientConfiguration, zkc);
        LedgerManagerFactory ledgerManagerFactory = AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, bkc.getMetadataClientDriver().getLayoutManager());
        ledgerManager = ledgerManagerFactory.newLedgerManager();
        this.bookieLedgerIndexer = new BookieLedgerIndexer(ledgerManager);
        this.ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager();
        this.admin = new BookKeeperAdmin(bkc, statsLogger);
        if (this.ledgerUnderreplicationManager.initializeLostBookieRecoveryDelay(conf.getLostBookieRecoveryDelay())) {
            LOG.info("Initializing lostBookieRecoveryDelay zNode to the conif value: {}", conf.getLostBookieRecoveryDelay());
        } else {
            LOG.info("Valid lostBookieRecoveryDelay zNode is available, so not creating " + "lostBookieRecoveryDelay zNode as part of Auditor initialization ");
        }
        lostBookieRecoveryDelayBeforeChange = this.ledgerUnderreplicationManager.getLostBookieRecoveryDelay();
    } catch (CompatibilityException ce) {
        throw new UnavailableException("CompatibilityException while initializing Auditor", ce);
    } catch (IOException | BKException | KeeperException ioe) {
        throw new UnavailableException("Exception while initializing Auditor", ioe);
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        throw new UnavailableException("Interrupted while initializing Auditor", ie);
    }
}

Example 7 with BookKeeperAdmin

use of org.apache.bookkeeper.client.BookKeeperAdmin in project pravega by pravega.

the class BookKeeperCleanupCommand method execute.

@Override
public void execute() throws Exception {
    ensureArgCount(0);
    @Cleanup val context = createContext();
    // Get all BK ledger ids.
    output("Searching for all the ledgers ...");
    @Cleanup val bkAdmin = new BookKeeperAdmin((BookKeeper) context.logFactory.getBookKeeperClient());
    val allLedgerIds = new ArrayList<Long>();
    bkAdmin.listLedgers().forEach(allLedgerIds::add);
    output("Searching for all referenced ledgers ...");
    // We will not be deleting any ledger id above the highest referenced Ledger Id since we may be inadvertently
    // deleting freshly created Ledgers that have not yet been added to a Ledger Metadata yet (BookKeeperLog.initialize
    // first creates a new Ledger and then updates the Metadata in ZK with its existence).
    AtomicLong highestReferencedLedgerId = new AtomicLong();
    val referencedLedgerIds = new HashSet<Long>();
    collectAllReferencedLedgerIds(referencedLedgerIds, context);
    highestReferencedLedgerId.set(referencedLedgerIds.stream().max(Long::compareTo).orElse(-1L));
    // We want to do our due diligence and verify there are no more other BookKeeperLogs that the user hasn't told us about.
    output("Searching for possible other BookKeeperLogs ...");
    checkForExtraLogs(context);
    // Determine deletion candidates.
    val deletionCandidates = allLedgerIds.stream().filter(id -> id < highestReferencedLedgerId.get() && !referencedLedgerIds.contains(id)).collect(Collectors.toList());
    output("\nTotal Count: %d, Referenced Count: %d, Highest Referenced Id: %s, To Delete Count: %d.", allLedgerIds.size(), referencedLedgerIds.size(), highestReferencedLedgerId, deletionCandidates.size());
    if (deletionCandidates.isEmpty()) {
        output("There are no Ledgers eligible for deletion at this time.");
        return;
    }
    output("\nDeletion candidates:");
    listCandidates(deletionCandidates, context);
    if (!confirmContinue()) {
        output("Not deleting anything at this time.");
        return;
    }
    // Search again for referenced ledger ids, in case any new ones were just referenced.
    collectAllReferencedLedgerIds(referencedLedgerIds, context);
    highestReferencedLedgerId.set(referencedLedgerIds.stream().max(Long::compareTo).orElse(-1L));
    deleteCandidates(deletionCandidates, referencedLedgerIds, context);
}

Example 8 with BookKeeperAdmin

use of org.apache.bookkeeper.client.BookKeeperAdmin in project pravega by pravega.

the class ContainerCommand method createContext.

/**
 * Creates a new Context to be used by the BookKeeper command.
 *
 * @return A new Context.
 * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized.
 */
@Override
public Context createContext() throws DurableDataLogException {
    val serviceConfig = getServiceConfig();
    val containerConfig = getCommandArgs().getState().getConfigBuilder().build().getConfig(ContainerConfig::builder);
    val bkConfig = getCommandArgs().getState().getConfigBuilder().include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL())).build().getConfig(BookKeeperConfig::builder);
    val zkClient = createZKClient();
    val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor());
    try {
        factory.initialize();
    } catch (DurableDataLogException ex) {
        zkClient.close();
        throw ex;
    }
    val bkAdmin = new BookKeeperAdmin((BookKeeper) factory.getBookKeeperClient());
    return new Context(serviceConfig, containerConfig, bkConfig, zkClient, factory, bkAdmin);
}

Example 9 with BookKeeperAdmin

use of org.apache.bookkeeper.client.BookKeeperAdmin in project pravega by pravega.

the class BookKeeperLogTests method testReconcileMetadata.

@Test
public void testReconcileMetadata() throws Exception {
    @Cleanup BookKeeperAdmin a = new BookKeeperAdmin((org.apache.bookkeeper.client.BookKeeper) this.factory.get().getBookKeeperClient());
    val initialLedgers = Sets.newHashSet(a.listLedgers());
    // Test initialization (node creation).
    try (val log = new TestBookKeeperLog()) {
        // Data not persisted and we throw an error - this is a real fencing event.
        log.setThrowZkException(true);
        log.setPersistData(false);
        AssertExtensions.assertThrows("Create(Persist=False, Throw=True)", () -> log.initialize(TIMEOUT), ex -> ex instanceof DataLogWriterNotPrimaryException);
        Assert.assertEquals(1, log.getCreateExceptionCount());
        // Data persisted correctly and we throw an error - reconciliation needed.
        log.setPersistData(true);
        log.initialize(TIMEOUT);
        Assert.assertEquals("Create(Persist=True, Throw=True)", 2, log.getCreateExceptionCount());
    }
    val expectedLedgerIds = new HashSet<Long>();
    // Test updates (subsequent recoveries).
    try (val log = new TestBookKeeperLog()) {
        // Data not persisted and we throw an error - this is a real fencing event.
        log.setThrowZkException(true);
        log.setPersistData(false);
        AssertExtensions.assertThrows("Update(Persist=False, Throw=True)", () -> log.initialize(TIMEOUT), ex -> ex instanceof DataLogWriterNotPrimaryException);
        Assert.assertEquals(1, log.getUpdateExceptionCount());
        // Data persisted correctly and we throw an error - reconciliation needed.
        log.setPersistData(true);
        log.initialize(TIMEOUT);
        Assert.assertEquals("Update(Persist=True, Throw=True)", 2, log.getUpdateExceptionCount());
        log.loadMetadata().getLedgers().stream().map(LedgerMetadata::getLedgerId).forEach(expectedLedgerIds::add);
    }
    // Verify ledger cleanup.
    val allLedgers = Sets.newHashSet(a.listLedgers());
    allLedgers.removeAll(initialLedgers);
    Assert.assertEquals("Unexpected ledgers in BK.", expectedLedgerIds, allLedgers);
}

Example 10 with BookKeeperAdmin

use of org.apache.bookkeeper.client.BookKeeperAdmin in project pravega by pravega.

the class BookKeeperCleanupCommand method execute.

@Override
public void execute() throws Exception {
    ensureArgCount(0);
    @Cleanup val context = createContext();
    // Get all BK ledger ids.
    output("Searching for all the ledgers ...");
    val bkAdmin = new BookKeeperAdmin(context.logFactory.getBookKeeperClient());
    val allLedgerIds = new ArrayList<Long>();
    bkAdmin.listLedgers().forEach(allLedgerIds::add);
    output("Searching for all referenced ledgers ...");
    // We will not be deleting any ledger id above the highest referenced Ledger Id since we may be inadvertently
    // deleting freshly created Ledgers that have not yet been added to a Ledger Metadata yet (BookKeeperLog.initialize
    // first creates a new Ledger and then updates the Metadata in ZK with its existence).
    AtomicLong highestReferencedLedgerId = new AtomicLong();
    val referencedLedgerIds = new HashSet<Long>();
    collectAllReferencedLedgerIds(referencedLedgerIds, context);
    highestReferencedLedgerId.set(referencedLedgerIds.stream().max(Long::compareTo).orElse(-1L));
    // We want to do our due diligence and verify there are no more other BookKeeperLogs that the user hasn't told us about.
    output("Searching for possible other BookKeeperLogs ...");
    checkForExtraLogs(context);
    // Determine deletion candidates.
    val deletionCandidates = allLedgerIds.stream().filter(id -> id < highestReferencedLedgerId.get() && !referencedLedgerIds.contains(id)).collect(Collectors.toList());
    output("\nTotal Count: %d, Referenced Count: %d, Highest Referenced Id: %s, To Delete Count: %d.", allLedgerIds.size(), referencedLedgerIds.size(), highestReferencedLedgerId, deletionCandidates.size());
    if (deletionCandidates.isEmpty()) {
        output("There are no Ledgers eligible for deletion at this time.");
        return;
    }
    output("\nDeletion candidates:");
    listCandidates(deletionCandidates, context);
    if (!confirmContinue()) {
        output("Not deleting anything at this time.");
        return;
    }
    // Search again for referenced ledger ids, in case any new ones were just referenced.
    collectAllReferencedLedgerIds(referencedLedgerIds, context);
    highestReferencedLedgerId.set(referencedLedgerIds.stream().max(Long::compareTo).orElse(-1L));
    deleteCandidates(deletionCandidates, referencedLedgerIds, context);
}