Examples with DLInterruptedException com.twitter.distributedlog.exceptions.DLInterruptedException used on opensource projects

Example 16 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class DLAuditor method calculateLedgerSpaceUsage.

private long calculateLedgerSpaceUsage(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    final AtomicLong totalBytes = new AtomicLong(0);
    final AtomicLong totalEntries = new AtomicLong(0);
    final AtomicLong numLedgers = new AtomicLong(0);
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final SettableFuture<Void> doneFuture = SettableFuture.create();
    final BookKeeper bk = bkc.get();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(final Long lid, final AsyncCallback.VoidCallback cb) {
            numLedgers.incrementAndGet();
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    bk.asyncOpenLedgerNoRecovery(lid, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8), new org.apache.bookkeeper.client.AsyncCallback.OpenCallback() {

                        @Override
                        public void openComplete(int rc, LedgerHandle lh, Object ctx) {
                            final int cbRc;
                            if (BKException.Code.OK == rc) {
                                totalBytes.addAndGet(lh.getLength());
                                totalEntries.addAndGet(lh.getLastAddConfirmed() + 1);
                                cbRc = rc;
                            } else {
                                cbRc = BKException.Code.ZKException;
                            }
                            executorService.submit(new Runnable() {

                                @Override
                                public void run() {
                                    cb.processResult(cbRc, null, null);
                                }
                            });
                        }
                    }, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.set(null);
            } else {
                doneFuture.setException(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("calculated {} ledgers\n\ttotal bytes = {}\n\ttotal entries = {}", new Object[] { numLedgers.get(), totalBytes.get(), totalEntries.get() });
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on calculating ledger space : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to calculate ledger space : ", e.getCause());
        }
    }
    return totalBytes.get();
}

Example 17 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class DLAuditor method collectLedgersFromAllocator.

private void collectLedgersFromAllocator(final URI uri, final com.twitter.distributedlog.DistributedLogManagerFactory factory, final List<String> allocationPaths, final Set<Long> ledgers) throws IOException {
    final LinkedBlockingQueue<String> poolQueue = new LinkedBlockingQueue<String>();
    for (String allocationPath : allocationPaths) {
        String rootPath = uri.getPath() + "/" + allocationPath;
        try {
            List<String> pools = getZooKeeperClient(factory).get().getChildren(rootPath, false);
            for (String pool : pools) {
                poolQueue.add(rootPath + "/" + pool);
            }
        } catch (KeeperException e) {
            throw new ZKException("Failed to get list of pools from " + rootPath, e);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new DLInterruptedException("Interrupted on getting list of pools from " + rootPath, e);
        }
    }
    logger.info("Collecting ledgers from allocators for {} : {}", uri, poolQueue);
    executeAction(poolQueue, 10, new Action<String>() {

        @Override
        public void execute(String poolPath) throws IOException {
            try {
                collectLedgersFromPool(poolPath);
            } catch (InterruptedException e) {
                throw new DLInterruptedException("Interrupted on collecting ledgers from allocation pool " + poolPath, e);
            } catch (KeeperException e) {
                throw new ZKException("Failed to collect ledgers from allocation pool " + poolPath, e.code());
            }
        }

        private void collectLedgersFromPool(String poolPath) throws InterruptedException, ZooKeeperClient.ZooKeeperConnectionException, KeeperException {
            List<String> allocators = getZooKeeperClient(factory).get().getChildren(poolPath, false);
            for (String allocator : allocators) {
                String allocatorPath = poolPath + "/" + allocator;
                byte[] data = getZooKeeperClient(factory).get().getData(allocatorPath, false, new Stat());
                if (null != data && data.length > 0) {
                    try {
                        long ledgerId = DLUtils.bytes2LedgerId(data);
                        synchronized (ledgers) {
                            ledgers.add(ledgerId);
                        }
                    } catch (NumberFormatException nfe) {
                        logger.warn("Invalid ledger found in allocator path {} : ", allocatorPath, nfe);
                    }
                }
            }
        }
    });
    logger.info("Collected ledgers from allocators for {}.", uri);
}

Example 18 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class DLAuditor method collectLedgersFromBK.

/**
     * Find leak ledgers phase 1: collect ledgers set.
     */
private Set<Long> collectLedgersFromBK(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final Set<Long> ledgers = new HashSet<Long>();
    final SettableFuture<Void> doneFuture = SettableFuture.create();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(Long lid, final AsyncCallback.VoidCallback cb) {
            synchronized (ledgers) {
                ledgers.add(lid);
                if (0 == ledgers.size() % 1000) {
                    logger.info("Collected {} ledgers", ledgers.size());
                }
            }
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    cb.processResult(BKException.Code.OK, null, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.set(null);
            } else {
                doneFuture.setException(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("Collected total {} ledgers", ledgers.size());
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on collecting ledgers : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to collect ledgers : ", e.getCause());
        }
    }
    return ledgers;
}

Example 19 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class DLAuditor method executeAction.

static <T> void executeAction(final LinkedBlockingQueue<T> queue, final int numThreads, final Action<T> action) throws IOException {
    final CountDownLatch failureLatch = new CountDownLatch(1);
    final CountDownLatch doneLatch = new CountDownLatch(queue.size());
    final AtomicInteger numFailures = new AtomicInteger(0);
    final AtomicInteger completedThreads = new AtomicInteger(0);
    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    try {
        for (int i = 0; i < numThreads; i++) {
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    while (true) {
                        T item = queue.poll();
                        if (null == item) {
                            break;
                        }
                        try {
                            action.execute(item);
                        } catch (IOException ioe) {
                            logger.error("Failed to execute action on item '{}'", item, ioe);
                            numFailures.incrementAndGet();
                            failureLatch.countDown();
                            break;
                        }
                        doneLatch.countDown();
                    }
                    if (numFailures.get() == 0 && completedThreads.incrementAndGet() == numThreads) {
                        failureLatch.countDown();
                    }
                }
            });
        }
        try {
            failureLatch.await();
            if (numFailures.get() > 0) {
                throw new IOException("Encountered " + numFailures.get() + " failures on executing action.");
            }
            doneLatch.await();
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            logger.warn("Interrupted on executing action", ie);
            throw new DLInterruptedException("Interrupted on executing action", ie);
        }
    } finally {
        executorService.shutdown();
    }
}

Example 20 with DLInterruptedException

use of com.twitter.distributedlog.exceptions.DLInterruptedException in project distributedlog by twitter.

the class MaxLogSegmentSequenceNo method store.

synchronized void store(ZooKeeperClient zkc, String path, long logSegmentSeqNo) throws IOException {
    try {
        Stat stat = zkc.get().setData(path, DLUtils.serializeLogSegmentSequenceNumber(logSegmentSeqNo), getZkVersion());
        update(stat.getVersion(), logSegmentSeqNo);
    } catch (KeeperException ke) {
        throw new ZKException("Error writing max ledger sequence number " + logSegmentSeqNo + " to " + path + " : ", ke);
    } catch (ZooKeeperClient.ZooKeeperConnectionException zce) {
        throw new IOException("Error writing max ledger sequence number " + logSegmentSeqNo + " to " + path + " : ", zce);
    } catch (InterruptedException e) {
        throw new DLInterruptedException("Error writing max ledger sequence number " + logSegmentSeqNo + " to " + path + " : ", e);
    }
}