Examples with ParallelBatchImporter org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter used on opensource projects

Example 1 with ParallelBatchImporter

use of org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class CheckPointingLogRotationStressTesting method shouldBehaveCorrectlyUnderStress.

@Test
public void shouldBehaveCorrectlyUnderStress() throws Throwable {
    long durationInMinutes = parseLong(fromEnv("CHECK_POINT_LOG_ROTATION_STRESS_DURATION", DEFAULT_DURATION_IN_MINUTES));
    File storeDir = new File(fromEnv("CHECK_POINT_LOG_ROTATION_STORE_DIRECTORY", DEFAULT_STORE_DIR));
    long nodeCount = parseLong(fromEnv("CHECK_POINT_LOG_ROTATION_NODE_COUNT", DEFAULT_NODE_COUNT));
    int threads = parseInt(fromEnv("CHECK_POINT_LOG_ROTATION_WORKER_THREADS", DEFAULT_WORKER_THREADS));
    String pageCacheMemory = fromEnv("CHECK_POINT_LOG_ROTATION_PAGE_CACHE_MEMORY", DEFAULT_PAGE_CACHE_MEMORY);
    System.out.println("1/6\tBuilding initial store...");
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction()) {
        new ParallelBatchImporter(ensureExistsAndEmpty(storeDir), fileSystem, DEFAULT, NullLogService.getInstance(), ExecutionMonitors.defaultVisible(), Config.defaults()).doImport(new NodeCountInputs(nodeCount));
    }
    System.out.println("2/6\tStarting database...");
    GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder(storeDir);
    GraphDatabaseService db = builder.setConfig(GraphDatabaseSettings.pagecache_memory, pageCacheMemory).setConfig(GraphDatabaseSettings.keep_logical_logs, Settings.FALSE).setConfig(GraphDatabaseSettings.check_point_interval_time, CHECK_POINT_INTERVAL_MINUTES + "m").setConfig(GraphDatabaseFacadeFactory.Configuration.tracer, "timer").newGraphDatabase();
    System.out.println("3/6\tWarm up db...");
    try (Workload workload = new Workload(db, defaultRandomMutation(nodeCount, db), threads)) {
        // make sure to run at least one checkpoint during warmup
        long warmUpTimeMillis = TimeUnit.SECONDS.toMillis(CHECK_POINT_INTERVAL_MINUTES * 2);
        workload.run(warmUpTimeMillis, Workload.TransactionThroughput.NONE);
    }
    System.out.println("4/6\tStarting workload...");
    TransactionThroughputChecker throughput = new TransactionThroughputChecker();
    try (Workload workload = new Workload(db, defaultRandomMutation(nodeCount, db), threads)) {
        workload.run(TimeUnit.MINUTES.toMillis(durationInMinutes), throughput);
    }
    System.out.println("5/6\tShutting down...");
    db.shutdown();
    try {
        System.out.println("6/6\tPrinting stats and recorded timings...");
        TimerTransactionTracer.printStats(System.out);
        throughput.assertThroughput(System.out);
    } finally {
        System.out.println("Done.");
    }
    // let's cleanup disk space when everything went well
    FileUtils.deleteRecursively(storeDir);
}

Example 2 with ParallelBatchImporter

use of org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class StoreMigrator method migrateWithBatchImporter.

private void migrateWithBatchImporter(File storeDir, File migrationDir, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, MigrationProgressMonitor.Section progressMonitor, RecordFormats oldFormat, RecordFormats newFormat) throws IOException {
    prepareBatchImportMigration(storeDir, migrationDir, oldFormat, newFormat);
    boolean requiresDynamicStoreMigration = !newFormat.dynamic().equals(oldFormat.dynamic());
    boolean requiresPropertyMigration = !newFormat.property().equals(oldFormat.property()) || requiresDynamicStoreMigration;
    File badFile = new File(storeDir, Configuration.BAD_FILE_NAME);
    try (NeoStores legacyStore = instantiateLegacyStore(oldFormat, storeDir);
        RecordCursors nodeInputCursors = new RecordCursors(legacyStore);
        RecordCursors relationshipInputCursors = new RecordCursors(legacyStore);
        OutputStream badOutput = new BufferedOutputStream(new FileOutputStream(badFile, false))) {
        Configuration importConfig = new Configuration.Overridden(config);
        AdditionalInitialIds additionalInitialIds = readAdditionalIds(lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset);
        // We have to make sure to keep the token ids if we're migrating properties/labels
        BatchImporter importer = new ParallelBatchImporter(migrationDir.getAbsoluteFile(), fileSystem, pageCache, importConfig, logService, withDynamicProcessorAssignment(migrationBatchImporterMonitor(legacyStore, progressMonitor, importConfig), importConfig), additionalInitialIds, config, newFormat);
        InputIterable<InputNode> nodes = legacyNodesAsInput(legacyStore, requiresPropertyMigration, nodeInputCursors);
        InputIterable<InputRelationship> relationships = legacyRelationshipsAsInput(legacyStore, requiresPropertyMigration, relationshipInputCursors);
        importer.doImport(Inputs.input(nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), Collectors.badCollector(badOutput, 0)));
        // During migration the batch importer doesn't necessarily writes all entities, depending on
        // which stores needs migration. Node, relationship, relationship group stores are always written
        // anyways and cannot be avoided with the importer, but delete the store files that weren't written
        // (left empty) so that we don't overwrite those in the real store directory later.
        Collection<StoreFile> storesToDeleteFromMigratedDirectory = new ArrayList<>();
        storesToDeleteFromMigratedDirectory.add(StoreFile.NEO_STORE);
        if (!requiresPropertyMigration) {
            // We didn't migrate properties, so the property stores in the migrated store are just empty/bogus
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.PROPERTY_STORE, StoreFile.PROPERTY_STRING_STORE, StoreFile.PROPERTY_ARRAY_STORE));
        }
        if (!requiresDynamicStoreMigration) {
            // We didn't migrate labels (dynamic node labels) or any other dynamic store
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.NODE_LABEL_STORE, StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, StoreFile.SCHEMA_STORE));
        }
        StoreFile.fileOperation(DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values());
        // When migrating on a block device there might be some files only accessible via the page cache.
        try {
            Predicate<FileHandle> fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream().anyMatch(storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fileHandle.getFile().getName()));
            pageCache.streamFilesRecursive(migrationDir).filter(fileHandlePredicate).forEach(FileHandle.HANDLE_DELETE);
        } catch (NoSuchFileException e) {
        // This means that we had no files only present in the page cache, this is fine.
        }
    }
}

Example 3 with ParallelBatchImporter

use of org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class ImportTool method doImport.

public static void doImport(PrintStream out, PrintStream err, File storeDir, File logsDir, File badFile, FileSystemAbstraction fs, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, boolean enableStacktrace, Input input, Config dbConfig, OutputStream badOutput, org.neo4j.unsafe.impl.batchimport.Configuration configuration) throws IOException {
    boolean success;
    LifeSupport life = new LifeSupport();
    LogService logService = life.add(StoreLogService.inLogsDirectory(fs, logsDir));
    life.start();
    //TODO: add file watcher here?
    BatchImporter importer = new ParallelBatchImporter(storeDir, fs, configuration, logService, ExecutionMonitors.defaultVisible(), dbConfig);
    printOverview(storeDir, nodesFiles, relationshipsFiles, configuration, out);
    success = false;
    try {
        importer.doImport(input);
        success = true;
    } catch (Exception e) {
        throw andPrintError("Import error", e, enableStacktrace, err);
    } finally {
        Collector collector = input.badCollector();
        int numberOfBadEntries = collector.badEntries();
        collector.close();
        badOutput.close();
        if (numberOfBadEntries > 0) {
            out.println("There were bad entries which were skipped and logged into " + badFile.getAbsolutePath());
        }
        life.shutdown();
        if (!success) {
            try {
                StoreFile.fileOperation(FileOperation.DELETE, fs, storeDir, null, Iterables.<StoreFile, StoreFile>iterable(StoreFile.values()), false, ExistingTargetStrategy.FAIL, StoreFileType.values());
            } catch (IOException e) {
                err.println("Unable to delete store files after an aborted import " + e);
                if (enableStacktrace) {
                    e.printStackTrace();
                }
            }
        }
    }
}

Example 4 with ParallelBatchImporter

use of org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class CsvInputBatchImportIT method shouldImportDataComingFromCsvFiles.

@Test
public void shouldImportDataComingFromCsvFiles() throws Exception {
    // GIVEN
    BatchImporter importer = new ParallelBatchImporter(directory.graphDbDir(), fileSystemRule.get(), smallBatchSizeConfig(), NullLogService.getInstance(), invisible(), Config.empty());
    List<InputNode> nodeData = randomNodeData();
    List<InputRelationship> relationshipData = randomRelationshipData(nodeData);
    // WHEN
    boolean success = false;
    try {
        importer.doImport(csv(nodeDataAsFile(nodeData), relationshipDataAsFile(relationshipData), IdType.STRING, lowBufferSize(COMMAS), silentBadCollector(0), getRuntime().availableProcessors()));
        // THEN
        verifyImportedData(nodeData, relationshipData);
        success = true;
    } finally {
        if (!success) {
            System.err.println("Seed " + seed);
        }
    }
}

Example 5 with ParallelBatchImporter

use of org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class MultipleIndexPopulationStressIT method createRandomData.

private void createRandomData(int count) throws IOException {
    Config config = Config.empty();
    RecordFormats recordFormats = RecordFormatSelector.selectForConfig(config, NullLogProvider.getInstance());
    BatchImporter importer = new ParallelBatchImporter(directory.graphDbDir(), fileSystemRule.get(), DEFAULT, NullLogService.getInstance(), ExecutionMonitors.invisible(), EMPTY, config, recordFormats);
    importer.doImport(new RandomDataInput(count));
}