Examples with System.nanoTime java.lang.System.nanoTime used on opensource projects

Example 1 with System.nanoTime

use of java.lang.System.nanoTime in project neo4j-apoc-procedures by neo4j-contrib.

the class Periodic method iterateAndExecuteBatchedInSeparateThread.

private Stream<BatchAndTotalResult> iterateAndExecuteBatchedInSeparateThread(int batchsize, boolean parallel, boolean iterateList, long retries, Iterator<Map<String, Object>> iterator, Consumer<Map<String, Object>> consumer) {
    ExecutorService pool = parallel ? Pools.DEFAULT : Pools.SINGLE;
    List<Future<Long>> futures = new ArrayList<>(1000);
    long batches = 0;
    long start = System.nanoTime();
    AtomicLong count = new AtomicLong();
    AtomicInteger failedOps = new AtomicInteger();
    AtomicLong retried = new AtomicLong();
    Map<String, Long> operationErrors = new ConcurrentHashMap<>();
    AtomicInteger failedBatches = new AtomicInteger();
    Map<String, Long> batchErrors = new HashMap<>();
    long successes = 0;
    do {
        if (Util.transactionIsTerminated(terminationGuard))
            break;
        if (log.isDebugEnabled())
            log.debug("execute in batch no " + batches + " batch size " + batchsize);
        List<Map<String, Object>> batch = Util.take(iterator, batchsize);
        long currentBatchSize = batch.size();
        Callable<Long> task;
        if (iterateList) {
            task = () -> {
                long c = count.addAndGet(currentBatchSize);
                if (Util.transactionIsTerminated(terminationGuard))
                    return 0L;
                List<Map<String, Object>> batchLocal = batch;
                try {
                    Map<String, Object> params = Util.map("_count", c, "_batch", batchLocal);
                    retried.addAndGet(retry(consumer, params, 0, retries));
                } catch (Exception e) {
                    failedOps.addAndGet(batchsize);
                    recordError(operationErrors, e);
                }
                return currentBatchSize;
            };
        } else {
            task = () -> {
                if (Util.transactionIsTerminated(terminationGuard))
                    return 0L;
                return batch.stream().map(p -> {
                    long c = count.incrementAndGet();
                    if (c % 1000 == 0 && Util.transactionIsTerminated(terminationGuard))
                        return 0;
                    List<Map<String, Object>> batchLocal = batch;
                    try {
                        Map<String, Object> params = merge(p, Util.map("_count", c, "_batch", batchLocal));
                        retried.addAndGet(retry(consumer, params, 0, retries));
                    } catch (Exception e) {
                        failedOps.incrementAndGet();
                        recordError(operationErrors, e);
                    }
                    return 1;
                }).mapToLong(l -> l).sum();
            };
        }
        futures.add(Util.inTxFuture(pool, db, task));
        batches++;
        if (futures.size() > 50) {
            while (futures.stream().filter(Future::isDone).count() == 0) {
                // none done yet, block for a bit
                LockSupport.parkNanos(1000);
            }
            Iterator<Future<Long>> it = futures.iterator();
            while (it.hasNext()) {
                Future<Long> future = it.next();
                if (future.isDone()) {
                    successes += Util.getFuture(future, batchErrors, failedBatches, 0L);
                    it.remove();
                }
            }
        }
    } while (iterator.hasNext());
    boolean wasTerminated = Util.transactionIsTerminated(terminationGuard);
    if (wasTerminated) {
        successes += futures.stream().mapToLong(f -> Util.getFutureOrCancel(f, batchErrors, failedBatches, 0L)).sum();
    } else {
        successes += futures.stream().mapToLong(f -> Util.getFuture(f, batchErrors, failedBatches, 0L)).sum();
    }
    Util.logErrors("Error during iterate.commit:", batchErrors, log);
    Util.logErrors("Error during iterate.execute:", operationErrors, log);
    long timeTaken = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start);
    BatchAndTotalResult result = new BatchAndTotalResult(batches, count.get(), timeTaken, successes, failedOps.get(), failedBatches.get(), retried.get(), operationErrors, batchErrors, wasTerminated);
    return Stream.of(result);
}