Examples with DLInterruptedException com.twitter.distributedlog.exceptions.DLInterruptedException used on opensource projects

Example 6 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class TestNonBlockingReadsMultiReader method testMultiReaders.

@Test(timeout = 60000)
public void testMultiReaders() throws Exception {
    String name = "distrlog-multireaders";
    final RateLimiter limiter = RateLimiter.create(1000);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    DistributedLogManager dlmwrite = createNewDLM(confLocal, name);
    final AsyncLogWriter writer = dlmwrite.startAsyncLogSegmentNonPartitioned();
    FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(0)));
    FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(1)));
    final AtomicInteger writeCount = new AtomicInteger(2);
    DistributedLogManager dlmread = createNewDLM(conf, name);
    BKSyncLogReaderDLSN reader0 = (BKSyncLogReaderDLSN) dlmread.getInputStream(0);
    try {
        ReaderThread[] readerThreads = new ReaderThread[1];
        readerThreads[0] = new ReaderThread("reader0-non-blocking", reader0, false);
        // readerThreads[1] = new ReaderThread("reader1-non-blocking", reader0, false);
        final AtomicBoolean running = new AtomicBoolean(true);
        Thread writerThread = new Thread("WriteThread") {

            @Override
            public void run() {
                try {
                    long txid = 2;
                    DLSN dlsn = DLSN.InvalidDLSN;
                    while (running.get()) {
                        limiter.acquire();
                        long curTxId = txid++;
                        dlsn = FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(curTxId)));
                        writeCount.incrementAndGet();
                        if (curTxId % 1000 == 0) {
                            LOG.info("writer write {}", curTxId);
                        }
                    }
                    LOG.info("Completed writing record at {}", dlsn);
                    Utils.close(writer);
                } catch (DLInterruptedException die) {
                    Thread.currentThread().interrupt();
                } catch (IOException e) {
                }
            }
        };
        for (ReaderThread rt : readerThreads) {
            rt.start();
        }
        writerThread.start();
        TimeUnit.SECONDS.sleep(5);
        LOG.info("Stopping writer");
        running.set(false);
        writerThread.join();
        LOG.info("Writer stopped after writing {} records, waiting for reader to complete", writeCount.get());
        while (writeCount.get() > (readerThreads[0].getReadCount())) {
            LOG.info("Write Count = {}, Read Count = {}, ReadAhead = {}", new Object[] { writeCount.get(), readerThreads[0].getReadCount(), reader0.getReadAheadPosition() });
            TimeUnit.MILLISECONDS.sleep(100);
        }
        assertEquals(writeCount.get(), (readerThreads[0].getReadCount()));
        for (ReaderThread readerThread : readerThreads) {
            readerThread.stopReading();
        }
    } finally {
        dlmwrite.close();
        reader0.close();
        dlmread.close();
    }
}

Example 7 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class DLAuditor method collectLedgersFromDL.

/**
     * Find leak ledgers phase 2: collect ledgers from uris.
     */
private Set<Long> collectLedgersFromDL(List<URI> uris, List<List<String>> allocationPaths) throws IOException {
    final Set<Long> ledgers = new TreeSet<Long>();
    List<com.twitter.distributedlog.DistributedLogManagerFactory> factories = new ArrayList<com.twitter.distributedlog.DistributedLogManagerFactory>(uris.size());
    try {
        for (URI uri : uris) {
            factories.add(new com.twitter.distributedlog.DistributedLogManagerFactory(conf, uri));
        }
        final CountDownLatch doneLatch = new CountDownLatch(uris.size());
        final AtomicInteger numFailures = new AtomicInteger(0);
        ExecutorService executor = Executors.newFixedThreadPool(uris.size());
        try {
            int i = 0;
            for (com.twitter.distributedlog.DistributedLogManagerFactory factory : factories) {
                final com.twitter.distributedlog.DistributedLogManagerFactory dlFactory = factory;
                final URI uri = uris.get(i);
                final List<String> aps = allocationPaths.get(i);
                i++;
                executor.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            logger.info("Collecting ledgers from {} : {}", uri, aps);
                            collectLedgersFromAllocator(uri, dlFactory, aps, ledgers);
                            synchronized (ledgers) {
                                logger.info("Collected {} ledgers from allocators for {} : {} ", new Object[] { ledgers.size(), uri, ledgers });
                            }
                            collectLedgersFromDL(uri, dlFactory, ledgers);
                        } catch (IOException e) {
                            numFailures.incrementAndGet();
                            logger.info("Error to collect ledgers from DL : ", e);
                        }
                        doneLatch.countDown();
                    }
                });
            }
            try {
                doneLatch.await();
                if (numFailures.get() > 0) {
                    throw new IOException(numFailures.get() + " errors to collect ledgers from DL");
                }
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                logger.warn("Interrupted on collecting ledgers from DL : ", e);
                throw new DLInterruptedException("Interrupted on collecting ledgers from DL : ", e);
            }
        } finally {
            executor.shutdown();
        }
    } finally {
        for (com.twitter.distributedlog.DistributedLogManagerFactory factory : factories) {
            factory.close();
        }
    }
    return ledgers;
}

Example 8 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class DLAuditor method calculateStreamSpaceUsage.

private long calculateStreamSpaceUsage(final com.twitter.distributedlog.DistributedLogManagerFactory factory, final String stream) throws IOException {
    DistributedLogManager dlm = factory.createDistributedLogManager(stream, com.twitter.distributedlog.DistributedLogManagerFactory.ClientSharingOption.SharedClients);
    long totalBytes = 0;
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            try {
                LedgerHandle lh = getBookKeeperClient(factory).get().openLedgerNoRecovery(segment.getLedgerId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
                totalBytes += lh.getLength();
                lh.close();
            } catch (BKException e) {
                logger.error("Failed to open ledger {} : ", segment.getLedgerId(), e);
                throw new IOException("Failed to open ledger " + segment.getLedgerId(), e);
            } catch (InterruptedException e) {
                logger.warn("Interrupted on opening ledger {} : ", segment.getLedgerId(), e);
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLedgerId(), e);
            }
        }
    } finally {
        dlm.close();
    }
    return totalBytes;
}

Example 9 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class LedgerAllocatorPool method initializePool.

private void initializePool() throws IOException {
    try {
        List<String> allocators;
        try {
            allocators = zkc.get().getChildren(poolPath, false);
        } catch (KeeperException.NoNodeException e) {
            logger.info("Allocator Pool {} doesn't exist. Creating it.", poolPath);
            ZkUtils.createFullPathOptimistic(zkc.get(), poolPath, new byte[0], zkc.getDefaultACL(), CreateMode.PERSISTENT);
            allocators = zkc.get().getChildren(poolPath, false);
        }
        if (null == allocators) {
            allocators = new ArrayList<String>();
        }
        if (allocators.size() < corePoolSize) {
            createAllocators(corePoolSize - allocators.size());
            allocators = zkc.get().getChildren(poolPath, false);
        }
        initializeAllocators(allocators);
    } catch (InterruptedException ie) {
        throw new DLInterruptedException("Interrupted when ensuring " + poolPath + " created : ", ie);
    } catch (KeeperException ke) {
        throw new IOException("Encountered zookeeper exception when initializing pool " + poolPath + " : ", ke);
    }
}

Example 10 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class ZKMetadataAccessor method createOrUpdateMetadata.

/**
     * Creates or update the metadata stored at the node associated with the
     * name and URI
     * @param metadata opaque metadata to be stored for the node
     * @throws IOException
     */
@Override
public void createOrUpdateMetadata(byte[] metadata) throws IOException {
    checkClosedOrInError("createOrUpdateMetadata");
    String zkPath = getZKPath();
    LOG.debug("Setting application specific metadata on {}", zkPath);
    try {
        Stat currentStat = writerZKC.get().exists(zkPath, false);
        if (currentStat == null) {
            if (metadata.length > 0) {
                Utils.zkCreateFullPathOptimistic(writerZKC, zkPath, metadata, writerZKC.getDefaultACL(), CreateMode.PERSISTENT);
            }
        } else {
            writerZKC.get().setData(zkPath, metadata, currentStat.getVersion());
        }
    } catch (InterruptedException ie) {
        throw new DLInterruptedException("Interrupted on creating or updating container metadata", ie);
    } catch (Exception exc) {
        throw new IOException("Exception creating or updating container metadata", exc);
    }
}