Examples with QueueBasedSpliterator apoc.util.QueueBasedSpliterator used on opensource projects

Example 1 with QueueBasedSpliterator

use of apoc.util.QueueBasedSpliterator in project neo4j-apoc-procedures by neo4j-contrib.

the class ExportCypher method exportCypher.

private Stream<DataProgressInfo> exportCypher(@Name("file") String fileName, String source, SubGraph graph, ExportConfig c, boolean onlySchema) throws IOException {
    if (fileName != null)
        checkWriteAllowed();
    ProgressInfo progressInfo = new ProgressInfo(fileName, source, "cypher");
    progressInfo.batchSize = c.getBatchSize();
    ProgressReporter reporter = new ProgressReporter(null, null, progressInfo);
    boolean separatedFiles = !onlySchema && c.separateFiles();
    FileManagerFactory.ExportCypherFileManager cypherFileManager = FileManagerFactory.createFileManager(fileName, separatedFiles, c.streamStatements());
    if (c.streamStatements()) {
        Future<Boolean> future = null;
        try {
            final ArrayBlockingQueue<DataProgressInfo> queue = new ArrayBlockingQueue<>(100);
            ProgressReporter reporterWithConsumer = reporter.withConsumer((pi) -> queue.offer(pi == ProgressInfo.EMPTY ? DataProgressInfo.EMPTY : new DataProgressInfo(pi).enrich(cypherFileManager)));
            future = Util.inTxFuture(Pools.DEFAULT, db, () -> {
                doExport(graph, c, onlySchema, reporterWithConsumer, cypherFileManager);
                return true;
            });
            QueueBasedSpliterator<DataProgressInfo> spliterator = new QueueBasedSpliterator<>(queue, DataProgressInfo.EMPTY, terminationGuard);
            return StreamSupport.stream(spliterator, false);
        } finally {
            Util.waitForFutures(Collections.singletonList(future));
        }
    } else {
        doExport(graph, c, onlySchema, reporter, cypherFileManager);
        return reporter.stream().map(DataProgressInfo::new).map((dpi) -> dpi.enrich(cypherFileManager));
    }
}

Example 2 with QueueBasedSpliterator

use of apoc.util.QueueBasedSpliterator in project neo4j-apoc-procedures by neo4j-contrib.

the class Cypher method mapParallel2.

@Procedure
@Description("apoc.cypher.mapParallel2(fragment, params, list-to-parallelize) yield value - executes fragment in parallel batches with the list segments being assigned to _")
public Stream<MapResult> mapParallel2(@Name("fragment") String fragment, @Name("params") Map<String, Object> params, @Name("list") List<Object> data, @Name("partitions") long partitions) {
    final String statement = withParamsAndIterator(fragment, params.keySet(), "_");
    db.execute("EXPLAIN " + statement).close();
    BlockingQueue<RowResult> queue = new ArrayBlockingQueue<>(100000);
    Stream<List<Object>> parallelPartitions = Util.partitionSubList(data, (int) (partitions <= 0 ? PARTITIONS : partitions), null);
    Util.inFuture(() -> {
        long total = parallelPartitions.map((List<Object> partition) -> {
            try {
                return executeStatement(queue, statement, parallelParams(params, "_", partition), false);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }).count();
        queue.put(RowResult.TOMBSTONE);
        return total;
    });
    return StreamSupport.stream(new QueueBasedSpliterator<>(queue, RowResult.TOMBSTONE, terminationGuard), true).map((rowResult) -> new MapResult(rowResult.result));
}