Examples with ParallelBatchImporter org.neo4j.internal.batchimport.ParallelBatchImporter used on opensource projects

Example 1 with ParallelBatchImporter

use of org.neo4j.internal.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class HumanUnderstandableExecutionMonitorIT method shouldReportProgressOfNodeImport.

@Test
void shouldReportProgressOfNodeImport() throws Exception {
    // given
    CapturingMonitor progress = new CapturingMonitor();
    HumanUnderstandableExecutionMonitor monitor = new HumanUnderstandableExecutionMonitor(progress);
    IdType idType = IdType.INTEGER;
    Input input = new DataGeneratorInput(NODE_COUNT, RELATIONSHIP_COUNT, idType, random.seed(), 0, bareboneNodeHeader(idType, new Extractors(';')), bareboneRelationshipHeader(idType, new Extractors(';')), 1, 1, 0, 0);
    Configuration configuration = new Configuration.Overridden(Configuration.DEFAULT) {

        @Override
        public long pageCacheMemory() {
            return mebiBytes(8);
        }
    };
    // when
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        new ParallelBatchImporter(databaseLayout, fileSystem, NULL, configuration, NullLogService.getInstance(), monitor, EMPTY, defaults(), LATEST_RECORD_FORMATS, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE).doImport(input);
        // then
        progress.assertAllProgressReachedEnd();
    }
}

Example 2 with ParallelBatchImporter

use of org.neo4j.internal.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class CsvInputEstimateCalculationIT method shouldCalculateCorrectEstimates.

@Test
void shouldCalculateCorrectEstimates() throws Exception {
    // given a couple of input files of various layouts
    Input input = generateData();
    RecordFormats format = LATEST_RECORD_FORMATS;
    Input.Estimates estimates = input.calculateEstimates(new PropertyValueRecordSizeCalculator(format.property().getRecordSize(NO_STORE_HEADER), GraphDatabaseInternalSettings.string_block_size.defaultValue(), 0, GraphDatabaseInternalSettings.array_block_size.defaultValue(), 0));
    // when
    Config config = Config.defaults();
    FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        new ParallelBatchImporter(databaseLayout, fs, PageCacheTracer.NULL, PBI_CONFIG, NullLogService.getInstance(), INVISIBLE, EMPTY, config, format, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE).doImport(input);
        // then compare estimates with actual disk sizes
        SingleFilePageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
        try (PageCache pageCache = new MuninnPageCache(swapperFactory, jobScheduler, MuninnPageCache.config(1000));
            NeoStores stores = new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable()).openAllNeoStores()) {
            assertRoughlyEqual(estimates.numberOfNodes(), stores.getNodeStore().getNumberOfIdsInUse());
            assertRoughlyEqual(estimates.numberOfRelationships(), stores.getRelationshipStore().getNumberOfIdsInUse());
            assertRoughlyEqual(estimates.numberOfNodeProperties() + estimates.numberOfRelationshipProperties(), calculateNumberOfProperties(stores));
        }
        long measuredPropertyStorage = propertyStorageSize();
        long estimatedPropertyStorage = estimates.sizeOfNodeProperties() + estimates.sizeOfRelationshipProperties();
        assertThat(estimatedPropertyStorage).as("Estimated property storage size of %s must be within 10%% of the measured size of %s.", bytesToString(estimatedPropertyStorage), bytesToString(measuredPropertyStorage)).isCloseTo(measuredPropertyStorage, withPercentage(10.0));
    }
}

Example 3 with ParallelBatchImporter

use of org.neo4j.internal.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class MultipleIndexPopulationStressIT method createRandomData.

private void createRandomData(long nodeCount, long relCount) throws Exception {
    Config config = Config.defaults(neo4j_home, directory.homePath());
    RecordFormats recordFormats = RecordFormatSelector.selectForConfig(config, NullLogProvider.getInstance());
    try (RandomDataInput input = new RandomDataInput(nodeCount, relCount);
        JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        DatabaseLayout layout = Neo4jLayout.of(directory.homePath()).databaseLayout(DEFAULT_DATABASE_NAME);
        IndexImporterFactory indexImporterFactory = new IndexImporterFactoryImpl(config);
        BatchImporter importer = new ParallelBatchImporter(layout, fileSystemAbstraction, PageCacheTracer.NULL, DEFAULT, NullLogService.getInstance(), ExecutionMonitor.INVISIBLE, EMPTY, config, recordFormats, NO_MONITOR, jobScheduler, Collector.EMPTY, TransactionLogInitializer.getLogFilesInitializer(), indexImporterFactory, INSTANCE);
        importer.doImport(input);
    }
}

Example 4 with ParallelBatchImporter

use of org.neo4j.internal.batchimport.ParallelBatchImporter in project neo4j by neo4j.

the class CsvInputBatchImportIT method shouldImportDataComingFromCsvFiles.

@Test
void shouldImportDataComingFromCsvFiles() throws Exception {
    // GIVEN
    Config dbConfig = Config.newBuilder().set(db_timezone, LogTimeZone.SYSTEM).set(dense_node_threshold, 5).build();
    try (JobScheduler scheduler = new ThreadPoolJobScheduler()) {
        BatchImporter importer = new ParallelBatchImporter(databaseLayout, fileSystem, PageCacheTracer.NULL, smallBatchSizeConfig(), NullLogService.getInstance(), ExecutionMonitor.INVISIBLE, EMPTY, dbConfig, defaultFormat(), ImportLogic.NO_MONITOR, scheduler, Collector.EMPTY, TransactionLogInitializer.getLogFilesInitializer(), new IndexImporterFactoryImpl(dbConfig), INSTANCE);
        List<InputEntity> nodeData = randomNodeData();
        List<InputEntity> relationshipData = randomRelationshipData(nodeData);
        // WHEN
        importer.doImport(csv(nodeDataAsFile(nodeData), relationshipDataAsFile(relationshipData), IdType.STRING, lowBufferSize(COMMAS)));
        // THEN
        verifyImportedData(nodeData, relationshipData);
    }
}