Examples with BookKeeperClient com.twitter.distributedlog.BookKeeperClient used on opensource projects

Example 1 with BookKeeperClient

use of com.twitter.distributedlog.BookKeeperClient in project distributedlog by twitter.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    BookKeeperClient bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    factory.close();
}

Example 2 with BookKeeperClient

use of com.twitter.distributedlog.BookKeeperClient in project distributedlog by twitter.

the class DLAuditor method collectLedgers.

public Pair<Set<Long>, Set<Long>> collectLedgers(List<URI> uris, List<List<String>> allocationPaths) throws IOException {
    Preconditions.checkArgument(uris.size() > 0, "No uri provided to audit");
    String zkServers = validateAndGetZKServers(uris);
    RetryPolicy retryPolicy = new BoundExponentialBackoffRetryPolicy(conf.getZKRetryBackoffStartMillis(), conf.getZKRetryBackoffMaxMillis(), Integer.MAX_VALUE);
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().name("DLAuditor-ZK").zkServers(zkServers).sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).retryPolicy(retryPolicy).zkAclId(conf.getZkAclId()).build();
    ExecutorService executorService = Executors.newCachedThreadPool();
    try {
        BKDLConfig bkdlConfig = resolveBKDLConfig(zkc, uris);
        logger.info("Resolved bookkeeper config : {}", bkdlConfig);
        BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder().name("DLAuditor-BK").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath()).build();
        try {
            Set<Long> bkLedgers = collectLedgersFromBK(bkc, executorService);
            Set<Long> dlLedgers = collectLedgersFromDL(uris, allocationPaths);
            return Pair.of(bkLedgers, dlLedgers);
        } finally {
            bkc.close();
        }
    } finally {
        zkc.close();
        executorService.shutdown();
    }
}

Example 3 with BookKeeperClient

use of com.twitter.distributedlog.BookKeeperClient in project distributedlog by twitter.

the class DLAuditor method calculateLedgerSpaceUsage.

public long calculateLedgerSpaceUsage(URI uri) throws IOException {
    List<URI> uris = Lists.newArrayList(uri);
    String zkServers = validateAndGetZKServers(uris);
    RetryPolicy retryPolicy = new BoundExponentialBackoffRetryPolicy(conf.getZKRetryBackoffStartMillis(), conf.getZKRetryBackoffMaxMillis(), Integer.MAX_VALUE);
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().name("DLAuditor-ZK").zkServers(zkServers).sessionTimeoutMs(conf.getZKSessionTimeoutMilliseconds()).retryPolicy(retryPolicy).zkAclId(conf.getZkAclId()).build();
    ExecutorService executorService = Executors.newCachedThreadPool();
    try {
        BKDLConfig bkdlConfig = resolveBKDLConfig(zkc, uris);
        logger.info("Resolved bookkeeper config : {}", bkdlConfig);
        BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder().name("DLAuditor-BK").dlConfig(conf).zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath()).build();
        try {
            return calculateLedgerSpaceUsage(bkc, executorService);
        } finally {
            bkc.close();
        }
    } finally {
        zkc.close();
        executorService.shutdown();
    }
}