Examples with StageExecution org.neo4j.internal.batchimport.staging.StageExecution used on opensource projects

Example 1 with StageExecution

use of org.neo4j.internal.batchimport.staging.StageExecution in project neo4j by neo4j.

the class DataImporter method importData.

private static long importData(String title, Configuration configuration, InputIterable data, BatchingNeoStores stores, Supplier<EntityImporter> visitors, ExecutionMonitor executionMonitor, StatsProvider memoryStatsProvider) throws IOException {
    LongAdder roughEntityCountProgress = new LongAdder();
    int numRunners = configuration.maxNumberOfProcessors();
    ExecutorService pool = Executors.newFixedThreadPool(numRunners, new NamedThreadFactory(title + "Importer"));
    IoMonitor writeMonitor = new IoMonitor(stores.getIoTracer());
    ControllableStep step = new ControllableStep(title, roughEntityCountProgress, configuration, writeMonitor, memoryStatsProvider);
    StageExecution execution = new StageExecution(title, null, configuration, Collections.singletonList(step), 0);
    long startTime = currentTimeMillis();
    List<EntityImporter> importers = new ArrayList<>();
    try (InputIterator dataIterator = data.iterator()) {
        executionMonitor.start(execution);
        for (int i = 0; i < numRunners; i++) {
            EntityImporter importer = visitors.get();
            importers.add(importer);
            pool.submit(new ExhaustingEntityImporterRunnable(execution, dataIterator, importer, roughEntityCountProgress));
        }
        pool.shutdown();
        long nextWait = 0;
        try {
            while (!pool.awaitTermination(nextWait, TimeUnit.MILLISECONDS)) {
                executionMonitor.check(execution);
                nextWait = executionMonitor.nextCheckTime() - currentTimeMillis();
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new IOException(e);
        }
    }
    execution.assertHealthy();
    stores.markHighIds();
    importers.forEach(EntityImporter::freeUnusedIds);
    step.markAsCompleted();
    writeMonitor.stop();
    executionMonitor.end(execution, currentTimeMillis() - startTime);
    execution.assertHealthy();
    return roughEntityCountProgress.sum();
}