Examples with QueueDistributor org.neo4j.consistency.checking.full.QueueDistribution.QueueDistributor used on opensource projects

Example 1 with QueueDistributor

use of org.neo4j.consistency.checking.full.QueueDistribution.QueueDistributor in project neo4j by neo4j.

the class RecordDistributor method distributeRecords.

public static <RECORD> void distributeRecords(int numberOfThreads, String workerNames, int queueSize, Iterator<RECORD> records, final ProgressListener progress, RecordProcessor<RECORD> processor, QueueDistributor<RECORD> idDistributor) {
    if (!records.hasNext()) {
        return;
    }
    @SuppressWarnings("unchecked") final ArrayBlockingQueue<RECORD>[] recordQ = new ArrayBlockingQueue[numberOfThreads];
    final Workers<RecordCheckWorker<RECORD>> workers = new Workers<>(workerNames);
    final AtomicInteger idGroup = new AtomicInteger(-1);
    for (int threadId = 0; threadId < numberOfThreads; threadId++) {
        recordQ[threadId] = new ArrayBlockingQueue<>(queueSize);
        workers.start(new RecordCheckWorker<>(threadId, idGroup, recordQ[threadId], processor));
    }
    final int[] recsProcessed = new int[numberOfThreads];
    RecordConsumer<RECORD> recordConsumer = (record, qIndex) -> {
        recordQ[qIndex].put(record);
        recsProcessed[qIndex]++;
    };
    try {
        while (records.hasNext()) {
            try {
                // Put records into the queues using the queue distributor. Each Worker will pull and process.
                RECORD record = records.next();
                idDistributor.distribute(record, recordConsumer);
                progress.add(1);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
        // No more records to distribute, mark as done so that the workers will exit when no more records in queue.
        for (RecordCheckWorker<RECORD> worker : workers) {
            worker.done();
        }
        workers.awaitAndThrowOnError(RuntimeException.class);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Was interrupted while awaiting completion");
    }
}