Examples with ProgressListener org.neo4j.helpers.progress.ProgressListener used on opensource projects

Example 1 with ProgressListener

use of org.neo4j.helpers.progress.ProgressListener in project neo4j by neo4j.

the class RecordScannerTest method shouldProcessRecordsParallelAndUpdateProgress.

@Test
public void shouldProcessRecordsParallelAndUpdateProgress() throws Exception {
    // given
    ProgressMonitorFactory.MultiPartBuilder progressBuilder = mock(ProgressMonitorFactory.MultiPartBuilder.class);
    ProgressListener progressListener = mock(ProgressListener.class);
    when(progressBuilder.progressForPart(anyString(), anyLong())).thenReturn(progressListener);
    @SuppressWarnings("unchecked") BoundedIterable<Integer> store = mock(BoundedIterable.class);
    when(store.iterator()).thenReturn(asList(42, 75, 192).iterator());
    @SuppressWarnings("unchecked") RecordProcessor<Integer> recordProcessor = mock(RecordProcessor.class);
    RecordScanner<Integer> scanner = new ParallelRecordScanner<>("our test task", Statistics.NONE, 1, store, progressBuilder, recordProcessor, CacheAccess.EMPTY, QueueDistribution.ROUND_ROBIN);
    // when
    scanner.run();
    // then
    verifyProcessCloseAndDone(recordProcessor, store, progressListener);
}

Example 2 with ProgressListener

use of org.neo4j.helpers.progress.ProgressListener in project neo4j by neo4j.

the class ApplyTransactionsCommand method applyTransactions.

private long applyTransactions(File fromPath, GraphDatabaseAPI toDb, long fromTxExclusive, long toTxInclusive, PrintStream out) throws IOException, TransactionFailureException {
    DependencyResolver resolver = toDb.getDependencyResolver();
    TransactionRepresentationCommitProcess commitProcess = new TransactionRepresentationCommitProcess(resolver.resolveDependency(TransactionAppender.class), resolver.resolveDependency(StorageEngine.class));
    LifeSupport life = new LifeSupport();
    try (DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
        PageCache pageCache = StandalonePageCacheFactory.createPageCache(fileSystem)) {
        LogicalTransactionStore source = life.add(new ReadOnlyTransactionStore(pageCache, fileSystem, fromPath, new Monitors()));
        life.start();
        long lastAppliedTx = fromTxExclusive;
        // Some progress if there are more than a couple of transactions to apply
        ProgressListener progress = toTxInclusive - fromTxExclusive >= 100 ? textual(out).singlePart("Application progress", toTxInclusive - fromTxExclusive) : ProgressListener.NONE;
        try (IOCursor<CommittedTransactionRepresentation> cursor = source.getTransactions(fromTxExclusive + 1)) {
            while (cursor.next()) {
                CommittedTransactionRepresentation transaction = cursor.get();
                TransactionRepresentation transactionRepresentation = transaction.getTransactionRepresentation();
                try {
                    commitProcess.commit(new TransactionToApply(transactionRepresentation), NULL, EXTERNAL);
                    progress.add(1);
                } catch (final Throwable e) {
                    System.err.println("ERROR applying transaction " + transaction.getCommitEntry().getTxId());
                    throw e;
                }
                lastAppliedTx = transaction.getCommitEntry().getTxId();
                if (lastAppliedTx == toTxInclusive) {
                    break;
                }
            }
        }
        return lastAppliedTx;
    } finally {
        life.shutdown();
    }
}

Example 3 with ProgressListener

use of org.neo4j.helpers.progress.ProgressListener in project neo4j by neo4j.

the class CountsBuilderDecorator method checkCounts.

public void checkCounts(CountsAccessor counts, final ConsistencyReporter reporter, ProgressMonitorFactory progressFactory) {
    final int nodes = nodeCounts.uniqueSize();
    final int relationships = relationshipCounts.uniqueSize();
    final int total = nodes + relationships;
    final AtomicInteger nodeEntries = new AtomicInteger(0);
    final AtomicInteger relationshipEntries = new AtomicInteger(0);
    final ProgressListener listener = progressFactory.singlePart("Checking node and relationship counts", total);
    listener.started();
    counts.accept(new CountsVisitor.Adapter() {

        @Override
        public void visitNodeCount(int labelId, long count) {
            nodeEntries.incrementAndGet();
            reporter.forCounts(new CountsEntry(nodeKey(labelId), count), CHECK_NODE_COUNT);
            listener.add(1);
        }

        @Override
        public void visitRelationshipCount(int startLabelId, int relTypeId, int endLabelId, long count) {
            relationshipEntries.incrementAndGet();
            reporter.forCounts(new CountsEntry(relationshipKey(startLabelId, relTypeId, endLabelId), count), CHECK_RELATIONSHIP_COUNT);
            listener.add(1);
        }
    });
    reporter.forCounts(new CountsEntry(nodeKey(WILDCARD), nodeEntries.get()), CHECK_NODE_KEY_COUNT);
    reporter.forCounts(new CountsEntry(relationshipKey(WILDCARD, WILDCARD, WILDCARD), relationshipEntries.get()), CHECK_RELATIONSHIP_KEY_COUNT);
    listener.done();
}

Example 4 with ProgressListener

use of org.neo4j.helpers.progress.ProgressListener in project neo4j by neo4j.

the class RecordDistributor method distributeRecords.

public static <RECORD> void distributeRecords(int numberOfThreads, String workerNames, int queueSize, Iterator<RECORD> records, final ProgressListener progress, RecordProcessor<RECORD> processor, QueueDistributor<RECORD> idDistributor) {
    if (!records.hasNext()) {
        return;
    }
    @SuppressWarnings("unchecked") final ArrayBlockingQueue<RECORD>[] recordQ = new ArrayBlockingQueue[numberOfThreads];
    final Workers<RecordCheckWorker<RECORD>> workers = new Workers<>(workerNames);
    final AtomicInteger idGroup = new AtomicInteger(-1);
    for (int threadId = 0; threadId < numberOfThreads; threadId++) {
        recordQ[threadId] = new ArrayBlockingQueue<>(queueSize);
        workers.start(new RecordCheckWorker<>(threadId, idGroup, recordQ[threadId], processor));
    }
    final int[] recsProcessed = new int[numberOfThreads];
    RecordConsumer<RECORD> recordConsumer = (record, qIndex) -> {
        recordQ[qIndex].put(record);
        recsProcessed[qIndex]++;
    };
    try {
        while (records.hasNext()) {
            try {
                // Put records into the queues using the queue distributor. Each Worker will pull and process.
                RECORD record = records.next();
                idDistributor.distribute(record, recordConsumer);
                progress.add(1);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
        // No more records to distribute, mark as done so that the workers will exit when no more records in queue.
        for (RecordCheckWorker<RECORD> worker : workers) {
            worker.done();
        }
        workers.awaitAndThrowOnError(RuntimeException.class);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Was interrupted while awaiting completion");
    }
}

Example 5 with ProgressListener

use of org.neo4j.helpers.progress.ProgressListener in project neo4j by neo4j.

the class RecordScannerTest method shouldProcessRecordsSequentiallyAndUpdateProgress.

@Test
public void shouldProcessRecordsSequentiallyAndUpdateProgress() throws Exception {
    // given
    ProgressMonitorFactory.MultiPartBuilder progressBuilder = mock(ProgressMonitorFactory.MultiPartBuilder.class);
    ProgressListener progressListener = mock(ProgressListener.class);
    when(progressBuilder.progressForPart(anyString(), anyLong())).thenReturn(progressListener);
    @SuppressWarnings("unchecked") BoundedIterable<Integer> store = mock(BoundedIterable.class);
    when(store.iterator()).thenReturn(asList(42, 75, 192).iterator());
    @SuppressWarnings("unchecked") RecordProcessor<Integer> recordProcessor = mock(RecordProcessor.class);
    RecordScanner<Integer> scanner = new SequentialRecordScanner<>("our test task", Statistics.NONE, 1, store, progressBuilder, recordProcessor);
    // when
    scanner.run();
    // then
    verifyProcessCloseAndDone(recordProcessor, store, progressListener);
}