Examples with DistributedLogManager com.twitter.distributedlog.DistributedLogManager used on opensource projects

Example 11 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class DistributedLogAdmin method checkStream.

private static StreamCandidate checkStream(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String streamName, final ExecutorService executorService, final BookKeeperClient bkc, String digestpw) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        if (segments.isEmpty()) {
            return null;
        }
        List<Future<LogSegmentCandidate>> futures = new ArrayList<Future<LogSegmentCandidate>>(segments.size());
        for (LogSegmentMetadata segment : segments) {
            futures.add(checkLogSegment(streamName, segment, executorService, bkc, digestpw));
        }
        List<LogSegmentCandidate> segmentCandidates;
        try {
            segmentCandidates = Await.result(Future.collect(futures));
        } catch (Exception e) {
            throw new IOException("Failed on checking stream " + streamName, e);
        }
        StreamCandidate streamCandidate = new StreamCandidate(streamName);
        for (LogSegmentCandidate segmentCandidate : segmentCandidates) {
            if (null != segmentCandidate) {
                streamCandidate.addLogSegmentCandidate(segmentCandidate);
            }
        }
        if (streamCandidate.segmentCandidates.isEmpty()) {
            return null;
        }
        return streamCandidate;
    } finally {
        dlm.close();
    }
}

Example 12 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class DLAuditor method collectLedgersFromStream.

private List<Long> collectLedgersFromStream(com.twitter.distributedlog.DistributedLogManagerFactory factory, String stream, Set<Long> ledgers) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        List<Long> sLedgers = new ArrayList<Long>();
        for (LogSegmentMetadata segment : segments) {
            synchronized (ledgers) {
                ledgers.add(segment.getLedgerId());
            }
            sLedgers.add(segment.getLedgerId());
        }
        return sLedgers;
    } finally {
        dlm.close();
    }
}

Example 13 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class DLAuditor method calculateStreamSpaceUsage.

private long calculateStreamSpaceUsage(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String stream) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
    long totalBytes = 0;
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            try {
                LedgerHandle lh = getBookKeeperClient(factory).get().openLedgerNoRecovery(segment.getLedgerId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
                totalBytes += lh.getLength();
                lh.close();
            } catch (BKException e) {
                logger.error("Failed to open ledger {} : ", segment.getLedgerId(), e);
                throw new IOException("Failed to open ledger " + segment.getLedgerId(), e);
            } catch (InterruptedException e) {
                logger.warn("Interrupted on opening ledger {} : ", segment.getLedgerId(), e);
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLedgerId(), e);
            }
        }
    } finally {
        dlm.close();
    }
    return totalBytes;
}

Example 14 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class ReaderWorker method close.

@Override
public void close() throws IOException {
    this.running = false;
    for (AsyncLogReader reader : logReaders) {
        if (null != reader) {
            FutureUtils.result(reader.asyncClose());
        }
    }
    for (DistributedLogManager dlm : dlms) {
        if (null != dlm) {
            dlm.close();
        }
    }
    namespace.close();
    SchedulerUtils.shutdownScheduler(executorService, 2, TimeUnit.MINUTES);
    SchedulerUtils.shutdownScheduler(callbackExecutor, 2, TimeUnit.MINUTES);
    if (this.dlc != null) {
        this.dlc.close();
    }
    for (DLZkServerSet serverSet : serverSets) {
        serverSet.close();
    }
}

Example 15 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogTool method testToolCreateZkAclId.

@Test(timeout = 60000)
public void testToolCreateZkAclId() throws Exception {
    createStream(defaultUri, "0", "CreateAclStream", defaultPrivilegedZkAclId);
    try {
        DistributedLogManager dlm = DLMTestUtil.createNewDLM("0CreateAclStream", conf, defaultUri);
        DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000);
        dlm.close();
    } catch (ZKException ex) {
        assertEquals(KeeperException.Code.NOAUTH, ex.getKeeperExceptionCode());
    }
}