Examples with DLInterruptedException org.apache.distributedlog.exceptions.DLInterruptedException used on opensource projects

Example 11 with DLInterruptedException

use of org.apache.distributedlog.exceptions.DLInterruptedException in project bookkeeper by apache.

the class DLAuditor method collectLedgersFromBK.

/**
 * Find leak ledgers phase 1: collect ledgers set.
 */
private Set<Long> collectLedgersFromBK(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final Set<Long> ledgers = new HashSet<Long>();
    final CompletableFuture<Void> doneFuture = FutureUtils.createFuture();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(Long lid, final AsyncCallback.VoidCallback cb) {
            synchronized (ledgers) {
                ledgers.add(lid);
                if (0 == ledgers.size() % 1000) {
                    logger.info("Collected {} ledgers", ledgers.size());
                }
            }
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    cb.processResult(BKException.Code.OK, null, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.complete(null);
            } else {
                doneFuture.completeExceptionally(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("Collected total {} ledgers", ledgers.size());
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on collecting ledgers : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to collect ledgers : ", e.getCause());
        }
    }
    return ledgers;
}

Example 12 with DLInterruptedException

use of org.apache.distributedlog.exceptions.DLInterruptedException in project bookkeeper by apache.

the class DLAuditor method collectLedgersFromAllocator.

private void collectLedgersFromAllocator(final URI uri, final Namespace namespace, final List<String> allocationPaths, final Set<Long> ledgers) throws IOException {
    final LinkedBlockingQueue<String> poolQueue = new LinkedBlockingQueue<String>();
    for (String allocationPath : allocationPaths) {
        String rootPath = uri.getPath() + "/" + allocationPath;
        try {
            List<String> pools = getZooKeeperClient(namespace).get().getChildren(rootPath, false);
            for (String pool : pools) {
                poolQueue.add(rootPath + "/" + pool);
            }
        } catch (KeeperException e) {
            throw new ZKException("Failed to get list of pools from " + rootPath, e);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new DLInterruptedException("Interrupted on getting list of pools from " + rootPath, e);
        }
    }
    logger.info("Collecting ledgers from allocators for {} : {}", uri, poolQueue);
    executeAction(poolQueue, 10, new Action<String>() {

        @Override
        public void execute(String poolPath) throws IOException {
            try {
                collectLedgersFromPool(poolPath);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on collecting" + " ledgers from allocation pool " + poolPath, e);
            } catch (KeeperException e) {
                throw new ZKException("Failed to collect ledgers from allocation pool " + poolPath, e.code());
            }
        }

        private void collectLedgersFromPool(String poolPath) throws InterruptedException, ZooKeeperClient.ZooKeeperConnectionException, KeeperException {
            List<String> allocators = getZooKeeperClient(namespace).get().getChildren(poolPath, false);
            for (String allocator : allocators) {
                String allocatorPath = poolPath + "/" + allocator;
                byte[] data = getZooKeeperClient(namespace).get().getData(allocatorPath, false, new Stat());
                if (null != data && data.length > 0) {
                    try {
                        long ledgerId = DLUtils.bytes2LogSegmentId(data);
                        synchronized (ledgers) {
                            ledgers.add(ledgerId);
                        }
                    } catch (NumberFormatException nfe) {
                        logger.warn("Invalid ledger found in allocator path {} : ", allocatorPath, nfe);
                    }
                }
            }
        }
    });
    logger.info("Collected ledgers from allocators for {}.", uri);
}

Example 13 with DLInterruptedException

use of org.apache.distributedlog.exceptions.DLInterruptedException in project bookkeeper by apache.

the class DLAuditor method calculateStreamSpaceUsage.

private long calculateStreamSpaceUsage(final Namespace namespace, final String stream) throws IOException {
    DistributedLogManager dlm = namespace.openLog(stream);
    long totalBytes = 0;
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        for (LogSegmentMetadata segment : segments) {
            try {
                LedgerHandle lh = getBookKeeperClient(namespace).get().openLedgerNoRecovery(segment.getLogSegmentId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
                totalBytes += lh.getLength();
                lh.close();
            } catch (BKException e) {
                logger.error("Failed to open ledger {} : ", segment.getLogSegmentId(), e);
                throw new IOException("Failed to open ledger " + segment.getLogSegmentId(), e);
            } catch (InterruptedException e) {
                logger.warn("Interrupted on opening ledger {} : ", segment.getLogSegmentId(), e);
                Thread.currentThread().interrupt();
                throw new DLInterruptedException("Interrupted on opening ledger " + segment.getLogSegmentId(), e);
            }
        }
    } finally {
        dlm.close();
    }
    return totalBytes;
}

Example 14 with DLInterruptedException

use of org.apache.distributedlog.exceptions.DLInterruptedException in project bookkeeper by apache.

the class DLAuditor method executeAction.

static <T> void executeAction(final LinkedBlockingQueue<T> queue, final int numThreads, final Action<T> action) throws IOException {
    final CountDownLatch failureLatch = new CountDownLatch(1);
    final CountDownLatch doneLatch = new CountDownLatch(queue.size());
    final AtomicInteger numFailures = new AtomicInteger(0);
    final AtomicInteger completedThreads = new AtomicInteger(0);
    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    try {
        for (int i = 0; i < numThreads; i++) {
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    while (true) {
                        T item = queue.poll();
                        if (null == item) {
                            break;
                        }
                        try {
                            action.execute(item);
                        } catch (IOException ioe) {
                            logger.error("Failed to execute action on item '{}'", item, ioe);
                            numFailures.incrementAndGet();
                            failureLatch.countDown();
                            break;
                        }
                        doneLatch.countDown();
                    }
                    if (numFailures.get() == 0 && completedThreads.incrementAndGet() == numThreads) {
                        failureLatch.countDown();
                    }
                }
            });
        }
        try {
            failureLatch.await();
            if (numFailures.get() > 0) {
                throw new IOException("Encountered " + numFailures.get() + " failures on executing action.");
            }
            doneLatch.await();
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            logger.warn("Interrupted on executing action", ie);
            throw new DLInterruptedException("Interrupted on executing action", ie);
        }
    } finally {
        executorService.shutdown();
    }
}

Example 15 with DLInterruptedException

use of org.apache.distributedlog.exceptions.DLInterruptedException in project bookkeeper by apache.

the class DLAuditor method collectLedgersFromDL.

/**
 * Find leak ledgers phase 2: collect ledgers from uris.
 */
private Set<Long> collectLedgersFromDL(List<URI> uris, List<List<String>> allocationPaths) throws IOException {
    final Set<Long> ledgers = new TreeSet<Long>();
    List<Namespace> namespaces = new ArrayList<Namespace>(uris.size());
    try {
        for (URI uri : uris) {
            namespaces.add(NamespaceBuilder.newBuilder().conf(conf).uri(uri).build());
        }
        final CountDownLatch doneLatch = new CountDownLatch(uris.size());
        final AtomicInteger numFailures = new AtomicInteger(0);
        ExecutorService executor = Executors.newFixedThreadPool(uris.size());
        try {
            int i = 0;
            for (final Namespace namespace : namespaces) {
                final Namespace dlNamespace = namespace;
                final URI uri = uris.get(i);
                final List<String> aps = allocationPaths.get(i);
                i++;
                executor.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            logger.info("Collecting ledgers from {} : {}", uri, aps);
                            collectLedgersFromAllocator(uri, namespace, aps, ledgers);
                            synchronized (ledgers) {
                                logger.info("Collected {} ledgers from allocators for {} : {} ", new Object[] { ledgers.size(), uri, ledgers });
                            }
                            collectLedgersFromDL(uri, namespace, ledgers);
                        } catch (IOException e) {
                            numFailures.incrementAndGet();
                            logger.info("Error to collect ledgers from DL : ", e);
                        }
                        doneLatch.countDown();
                    }
                });
            }
            try {
                doneLatch.await();
                if (numFailures.get() > 0) {
                    throw new IOException(numFailures.get() + " errors to collect ledgers from DL");
                }
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                logger.warn("Interrupted on collecting ledgers from DL : ", e);
                throw new DLInterruptedException("Interrupted on collecting ledgers from DL : ", e);
            }
        } finally {
            executor.shutdown();
        }
    } finally {
        for (Namespace namespace : namespaces) {
            namespace.close();
        }
    }
    return ledgers;
}