Examples with Configuration org.neo4j.unsafe.impl.batchimport.Configuration used on opensource projects

Example 1 with Configuration

use of org.neo4j.unsafe.impl.batchimport.Configuration in project neo4j by neo4j.

the class CsvImporter method doImport.

@Override
public void doImport() throws IOException {
    FileSystemAbstraction fs = outsideWorld.fileSystem();
    File storeDir = config.get(DatabaseManagementSystemSettings.database_path);
    File logsDir = config.get(GraphDatabaseSettings.logs_directory);
    File reportFile = new File(reportFileName);
    OutputStream badOutput = new BufferedOutputStream(fs.openAsOutputStream(reportFile, false));
    Collector badCollector = badCollector(badOutput, isIgnoringSomething() ? BadCollector.UNLIMITED_TOLERANCE : 0, collect(ignoreBadRelationships, ignoreDuplicateNodes, ignoreExtraColumns));
    Configuration configuration = importConfiguration(null, false, config);
    CsvInput input = new CsvInput(nodeData(inputEncoding, nodesFiles), defaultFormatNodeFileHeader(), relationshipData(inputEncoding, relationshipsFiles), defaultFormatRelationshipFileHeader(), idType, csvConfiguration(args, false), badCollector, configuration.maxNumberOfProcessors());
    ImportTool.doImport(outsideWorld.errorStream(), outsideWorld.errorStream(), storeDir, logsDir, reportFile, fs, nodesFiles, relationshipsFiles, false, input, config, badOutput, configuration);
}

Example 2 with Configuration

use of org.neo4j.unsafe.impl.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldRemoveCPUsFromWayTooFastStep.

@Test
public void shouldRemoveCPUsFromWayTooFastStep() throws Exception {
    // GIVEN
    Configuration config = movingAverageConfig(10);
    // available processors = 2 is enough because it will see the fast step as only using 20% of a processor
    // and it rounds down. So there's room for assigning one more.
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config, 3);
    ControlledStep<?> slowStep = spy(stepWithStats("slow", 1, avg_processing_time, 6L, done_batches, 10L).setProcessors(2));
    ControlledStep<?> fastStep = spy(stepWithStats("fast", 0, avg_processing_time, 2L, done_batches, 10L).setProcessors(2));
    StageExecution execution = executionOf(config, slowStep, fastStep);
    assigner.start(execution);
    // WHEN checking
    assigner.check(execution);
    // THEN one processor should be removed from the fast step
    verify(fastStep, times(1)).processors(-1);
}

Example 3 with Configuration

use of org.neo4j.unsafe.impl.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldRemoveCPUsButNotSoThatTheFastStepBecomesBottleneck.

@Test
public void shouldRemoveCPUsButNotSoThatTheFastStepBecomesBottleneck() throws Exception {
    // GIVEN
    Configuration config = movingAverageConfig(10);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config, 3);
    ControlledStep<?> slowStep = spy(stepWithStats("slow", 1, avg_processing_time, 10L, done_batches, 10L));
    ControlledStep<?> fastStep = spy(stepWithStats("fast", 0, avg_processing_time, 7L, done_batches, 10L).setProcessors(3));
    StageExecution execution = executionOf(config, slowStep, fastStep);
    assigner.start(execution);
    // WHEN checking the first time
    assigner.check(execution);
    // THEN one processor should be removed from the fast step
    verify(fastStep, times(0)).processors(1);
    verify(fastStep, times(0)).processors(-1);
}

Example 4 with Configuration

use of org.neo4j.unsafe.impl.batchimport.Configuration in project neo4j by neo4j.

the class StoreMigrator method migrateWithBatchImporter.

private void migrateWithBatchImporter(File storeDir, File migrationDir, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, MigrationProgressMonitor.Section progressMonitor, RecordFormats oldFormat, RecordFormats newFormat) throws IOException {
    prepareBatchImportMigration(storeDir, migrationDir, oldFormat, newFormat);
    boolean requiresDynamicStoreMigration = !newFormat.dynamic().equals(oldFormat.dynamic());
    boolean requiresPropertyMigration = !newFormat.property().equals(oldFormat.property()) || requiresDynamicStoreMigration;
    File badFile = new File(storeDir, Configuration.BAD_FILE_NAME);
    try (NeoStores legacyStore = instantiateLegacyStore(oldFormat, storeDir);
        RecordCursors nodeInputCursors = new RecordCursors(legacyStore);
        RecordCursors relationshipInputCursors = new RecordCursors(legacyStore);
        OutputStream badOutput = new BufferedOutputStream(new FileOutputStream(badFile, false))) {
        Configuration importConfig = new Configuration.Overridden(config);
        AdditionalInitialIds additionalInitialIds = readAdditionalIds(lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset);
        // We have to make sure to keep the token ids if we're migrating properties/labels
        BatchImporter importer = new ParallelBatchImporter(migrationDir.getAbsoluteFile(), fileSystem, pageCache, importConfig, logService, withDynamicProcessorAssignment(migrationBatchImporterMonitor(legacyStore, progressMonitor, importConfig), importConfig), additionalInitialIds, config, newFormat);
        InputIterable<InputNode> nodes = legacyNodesAsInput(legacyStore, requiresPropertyMigration, nodeInputCursors);
        InputIterable<InputRelationship> relationships = legacyRelationshipsAsInput(legacyStore, requiresPropertyMigration, relationshipInputCursors);
        importer.doImport(Inputs.input(nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), Collectors.badCollector(badOutput, 0)));
        // During migration the batch importer doesn't necessarily writes all entities, depending on
        // which stores needs migration. Node, relationship, relationship group stores are always written
        // anyways and cannot be avoided with the importer, but delete the store files that weren't written
        // (left empty) so that we don't overwrite those in the real store directory later.
        Collection<StoreFile> storesToDeleteFromMigratedDirectory = new ArrayList<>();
        storesToDeleteFromMigratedDirectory.add(StoreFile.NEO_STORE);
        if (!requiresPropertyMigration) {
            // We didn't migrate properties, so the property stores in the migrated store are just empty/bogus
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.PROPERTY_STORE, StoreFile.PROPERTY_STRING_STORE, StoreFile.PROPERTY_ARRAY_STORE));
        }
        if (!requiresDynamicStoreMigration) {
            // We didn't migrate labels (dynamic node labels) or any other dynamic store
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.NODE_LABEL_STORE, StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, StoreFile.SCHEMA_STORE));
        }
        StoreFile.fileOperation(DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values());
        // When migrating on a block device there might be some files only accessible via the page cache.
        try {
            Predicate<FileHandle> fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream().anyMatch(storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fileHandle.getFile().getName()));
            pageCache.streamFilesRecursive(migrationDir).filter(fileHandlePredicate).forEach(FileHandle.HANDLE_DELETE);
        } catch (NoSuchFileException e) {
        // This means that we had no files only present in the page cache, this is fine.
        }
    }
}

Example 5 with Configuration

use of org.neo4j.unsafe.impl.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldRemoveCPUsFromTooFastStepEvenIfThereIsAWayFaster.

@Test
public void shouldRemoveCPUsFromTooFastStepEvenIfThereIsAWayFaster() throws Exception {
    // The point is that not only the fastest step is subject to have processors removed,
    // it's the relationship between all pairs of steps. This is important since the DPA has got
    // a max permit count of processors to assign, so reclaiming unnecessary assignments can
    // have those be assigned to a more appropriate step instead, where it will benefit the Stage more.
    // GIVEN
    Configuration config = movingAverageConfig(10);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config, 3);
    Step<?> wayFastest = stepWithStats("wayFastest", 0, avg_processing_time, 50L, done_batches, 20L);
    Step<?> fast = spy(stepWithStats("fast", 0, avg_processing_time, 100L, done_batches, 20L).setProcessors(3));
    Step<?> slow = stepWithStats("slow", 1, avg_processing_time, 220L, done_batches, 20L);
    StageExecution execution = executionOf(config, slow, wayFastest, fast);
    assigner.start(execution);
    // WHEN
    assigner.check(execution);
    // THEN
    verify(fast).processors(-1);
}