Examples with AbstractIterator com.google.common.collect.AbstractIterator used on opensource projects

Example 31 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project gatk by broadinstitute.

the class Utils method transformParallel.

/**
     * Like Guava's {@link Iterators#transform(Iterator, com.google.common.base.Function)}, but runs a fixed number
     * ({@code numThreads}) of transformations in parallel, while maintaining ordering of the output iterator.
     * This is useful if the transformations are CPU intensive.
     */
public static <F, T> Iterator<T> transformParallel(final Iterator<F> fromIterator, final Function<F, T> function, final int numThreads) {
    Utils.nonNull(fromIterator, "fromIterator");
    Utils.nonNull(function, "function");
    Utils.validateArg(numThreads >= 1, "numThreads must be at least 1");
    if (numThreads == 1) {
        // defer to Guava for single-threaded case
        return Iterators.transform(fromIterator, new com.google.common.base.Function<F, T>() {

            @Nullable
            @Override
            public T apply(@Nullable final F input) {
                return function.apply(input);
            }
        });
    }
    // use an executor service for the multi-threaded case
    final ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    final Queue<Future<T>> futures = new LinkedList<>();
    return new AbstractIterator<T>() {

        @Override
        protected T computeNext() {
            try {
                while (fromIterator.hasNext()) {
                    if (futures.size() == numThreads) {
                        return futures.remove().get();
                    }
                    final F next = fromIterator.next();
                    final Future<T> future = executorService.submit(() -> function.apply(next));
                    futures.add(future);
                }
                if (!futures.isEmpty()) {
                    return futures.remove().get();
                }
                executorService.shutdown();
                return endOfData();
            } catch (InterruptedException | ExecutionException e) {
                throw new GATKException("Problem running task", e);
            }
        }
    };
}

Example 32 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project GeoGig by boundlessgeo.

the class BinaryPackedChanges method asObjects.

/**
     * Returns an iterator that calls the {@code callback} for each {@link DiffPacket}'s
     * {@link DiffEntry} once, and returns either zero, one, or two {@link RevObject}s, depending on
     * which information the diff packet carried over.
     */
private Iterator<RevObject> asObjects(final PacketReadingIterator readingIterator, final Callback callback) {
    return new AbstractIterator<RevObject>() {

        private DiffPacket current;

        @Override
        protected RevObject computeNext() {
            if (current != null) {
                Preconditions.checkState(current.metadataObject != null);
                RevObject ret = current.metadataObject;
                current = null;
                return ret;
            }
            while (readingIterator.hasNext()) {
                DiffPacket diffPacket = readingIterator.next();
                callback.callback(diffPacket.entry);
                RevObject obj = diffPacket.newObject;
                RevObject md = diffPacket.metadataObject;
                Preconditions.checkState(obj != null || (obj == null && md == null));
                if (obj != null) {
                    if (md != null) {
                        current = diffPacket;
                    }
                    return obj;
                }
            }
            return endOfData();
        }
    };
}

Example 33 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project GeoGig by boundlessgeo.

the class ChangesetContentsScanner method parse.

public Iterator<Change> parse(InputStream changesetDownloadStream) throws XMLStreamException {
    final XMLStreamReader reader;
    reader = XMLInputFactory.newFactory().createXMLStreamReader(changesetDownloadStream, "UTF-8");
    // position reader at first change, if any
    reader.nextTag();
    reader.require(START_ELEMENT, null, "osmChange");
    Iterator<Change> iterator = new AbstractIterator<Change>() {

        @Override
        protected Change computeNext() {
            Change next;
            try {
                if (findNextChange(reader)) {
                    next = parseChange(reader);
                } else {
                    return super.endOfData();
                }
            } catch (XMLStreamException e) {
                System.err.println("Error parsing change, ignoring and continuing " + "with next change if possible. " + e.getMessage());
                next = computeNext();
            }
            return next;
        }
    };
    return iterator;
}

Example 34 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project GeoGig by boundlessgeo.

the class HistoryDownloader method parseChanges.

private Optional<Iterator<Change>> parseChanges(Supplier<Optional<File>> file) {
    final Optional<File> changesFile;
    try {
        changesFile = file.get();
    } catch (RuntimeException e) {
        Throwable cause = e.getCause();
        if (cause instanceof FileNotFoundException) {
            return Optional.absent();
        }
        throw Throwables.propagate(e);
    }
    if (!changesFile.isPresent()) {
        return Optional.absent();
    }
    final File actualFile = changesFile.get();
    final InputStream stream = openStream(actualFile);
    final Iterator<Change> changes;
    ChangesetContentsScanner scanner = new ChangesetContentsScanner();
    try {
        changes = scanner.parse(stream);
    } catch (XMLStreamException e) {
        throw Throwables.propagate(e);
    }
    Iterator<Change> iterator = new AbstractIterator<Change>() {

        @Override
        protected Change computeNext() {
            if (!changes.hasNext()) {
                Closeables.closeQuietly(stream);
                actualFile.delete();
                actualFile.getParentFile().delete();
                return super.endOfData();
            }
            return changes.next();
        }
    };
    return Optional.of(iterator);
}

Example 35 with AbstractIterator

use of com.google.common.collect.AbstractIterator in project GeoGig by boundlessgeo.

the class DiffTree method _call.

/**
     * Finds differences between the two specified trees.
     * 
     * @return an iterator to a set of differences between the two trees
     * @see DiffEntry
     */
@Override
protected Iterator<DiffEntry> _call() throws IllegalArgumentException {
    checkNotNull(oldRefSpec, "old version not specified");
    checkNotNull(newRefSpec, "new version not specified");
    final RevTree oldTree = resolveTree(oldRefSpec);
    final RevTree newTree = resolveTree(newRefSpec);
    if (oldTree.equals(newTree)) {
        return Iterators.emptyIterator();
    }
    ObjectDatabase leftSource = resolveSource(oldTree.getId());
    ObjectDatabase rightSource = resolveSource(newTree.getId());
    final PreOrderDiffWalk visitor = new PreOrderDiffWalk(oldTree, newTree, leftSource, rightSource);
    final BlockingQueue<DiffEntry> queue = new ArrayBlockingQueue<>(100);
    final DiffEntryProducer diffProducer = new DiffEntryProducer(queue);
    diffProducer.setReportTrees(this.reportTrees);
    diffProducer.setRecursive(this.recursive);
    final List<RuntimeException> producerErrors = new LinkedList<>();
    Thread producerThread = new Thread("DiffTree producer thread") {

        @Override
        public void run() {
            Consumer consumer = diffProducer;
            if (customFilter != null) {
                // evaluated the latest
                consumer = new PreOrderDiffWalk.FilteringConsumer(consumer, customFilter);
            }
            if (changeTypeFilter != null) {
                consumer = new ChangeTypeFilteringDiffConsumer(changeTypeFilter, consumer);
            }
            if (boundsFilter != null) {
                consumer = new BoundsFilteringDiffConsumer(boundsFilter, consumer, stagingDatabase());
            }
            if (!pathFilters.isEmpty()) {
                // evaluated the former
                consumer = new PathFilteringDiffConsumer(pathFilters, consumer);
            }
            try {
                visitor.walk(consumer);
            } catch (RuntimeException e) {
                LOGGER.error("Error traversing diffs", e);
                producerErrors.add(e);
            } finally {
                diffProducer.finished = true;
            }
        }
    };
    producerThread.setDaemon(true);
    producerThread.start();
    Iterator<DiffEntry> consumerIterator = new AbstractIterator<DiffEntry>() {

        @Override
        protected DiffEntry computeNext() {
            if (!producerErrors.isEmpty()) {
                throw new RuntimeException("Error in producer thread", producerErrors.get(0));
            }
            BlockingQueue<DiffEntry> entries = queue;
            boolean finished = diffProducer.isFinished();
            boolean empty = entries.isEmpty();
            while (!finished || !empty) {
                try {
                    DiffEntry entry = entries.poll(10, TimeUnit.MILLISECONDS);
                    if (entry != null) {
                        return entry;
                    }
                    finished = diffProducer.isFinished();
                    empty = entries.isEmpty();
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
            }
            return endOfData();
        }

        @Override
        protected void finalize() {
            diffProducer.finished = true;
        }
    };
    return consumerIterator;
}