Examples with CsvInput org.neo4j.internal.batchimport.input.csv.CsvInput used on opensource projects

Example 1 with CsvInput

use of org.neo4j.internal.batchimport.input.csv.CsvInput in project neo4j by neo4j.

the class ImportPanicIT method shouldExitAndThrowExceptionOnPanic.

/**
 * There was this problem where some steps and in particular parallel CSV input parsing that
 * paniced would hang the import entirely.
 */
@Test
void shouldExitAndThrowExceptionOnPanic() throws Exception {
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        BatchImporter importer = new ParallelBatchImporter(databaseLayout, testDirectory.getFileSystem(), PageCacheTracer.NULL, Configuration.DEFAULT, NullLogService.getInstance(), ExecutionMonitor.INVISIBLE, AdditionalInitialIds.EMPTY, Config.defaults(), StandardV3_4.RECORD_FORMATS, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE);
        Iterable<DataFactory> nodeData = DataFactories.datas(DataFactories.data(InputEntityDecorators.NO_DECORATOR, fileAsCharReadable(nodeCsvFileWithBrokenEntries())));
        Input brokenCsvInput = new CsvInput(nodeData, DataFactories.defaultFormatNodeFileHeader(), DataFactories.datas(), DataFactories.defaultFormatRelationshipFileHeader(), IdType.ACTUAL, csvConfigurationWithLowBufferSize(), CsvInput.NO_MONITOR, INSTANCE);
        var e = assertThrows(InputException.class, () -> importer.doImport(brokenCsvInput));
        assertTrue(e.getCause() instanceof DataAfterQuoteException);
    }
}

Example 2 with CsvInput

use of org.neo4j.internal.batchimport.input.csv.CsvInput in project neo4j by neo4j.

the class CsvImporter method doImport.

@Override
public void doImport() throws IOException {
    if (force) {
        fileSystem.deleteRecursively(databaseLayout.databaseDirectory());
        fileSystem.deleteRecursively(databaseLayout.getTransactionLogsDirectory());
    }
    try (OutputStream badOutput = fileSystem.openAsOutputStream(reportFile, false);
        Collector badCollector = getBadCollector(skipBadEntriesLogging, badOutput)) {
        // Extract the default time zone from the database configuration
        ZoneId dbTimeZone = databaseConfig.get(GraphDatabaseSettings.db_temporal_timezone);
        Supplier<ZoneId> defaultTimeZone = () -> dbTimeZone;
        final var nodeData = nodeData();
        final var relationshipsData = relationshipData();
        CsvInput input = new CsvInput(nodeData, defaultFormatNodeFileHeader(defaultTimeZone, normalizeTypes), relationshipsData, defaultFormatRelationshipFileHeader(defaultTimeZone, normalizeTypes), idType, csvConfig, new CsvInput.PrintingMonitor(stdOut), memoryTracker);
        doImport(input, badCollector);
    }
}