Examples with Input org.neo4j.unsafe.impl.batchimport.input.Input used on opensource projects

Example 1 with Input

use of org.neo4j.unsafe.impl.batchimport.input.Input in project neo4j by neo4j.

the class ImportPanicIT method shouldExitAndThrowExceptionOnPanic.

/**
     * There was this problem where some steps and in particular parallel CSV input parsing that
     * paniced would hang the import entirely.
     */
@Test
public void shouldExitAndThrowExceptionOnPanic() throws Exception {
    // GIVEN
    BatchImporter importer = new ParallelBatchImporter(directory.absolutePath(), fs, Configuration.DEFAULT, NullLogService.getInstance(), ExecutionMonitors.invisible(), AdditionalInitialIds.EMPTY, Config.empty(), StandardV3_0.RECORD_FORMATS);
    Iterable<DataFactory<InputNode>> nodeData = nodeData(data(NO_NODE_DECORATOR, fileAsCharReadable(nodeCsvFileWithBrokenEntries())));
    Input brokenCsvInput = new CsvInput(nodeData, defaultFormatNodeFileHeader(), relationshipData(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, csvConfigurationWithLowBufferSize(), new BadCollector(new NullOutputStream(), 0, 0), Runtime.getRuntime().availableProcessors());
    // WHEN
    try {
        importer.doImport(brokenCsvInput);
        fail("Should have failed properly");
    } catch (InputException e) {
        // THEN
        assertTrue(e.getCause() instanceof DataAfterQuoteException);
    // and we managed to shut down properly
    }
}

Example 2 with Input

use of org.neo4j.unsafe.impl.batchimport.input.Input in project neo4j by neo4j.

the class ExternalPropertiesDecoratorIT method shouldDecorateExternalPropertiesInParallelProcessingCsvInput.

@Test
public void shouldDecorateExternalPropertiesInParallelProcessingCsvInput() throws Exception {
    // GIVEN
    int processors = 5;
    Collector collector = mock(Collector.class);
    int count = 1000;
    Configuration config = new Configuration.Overridden(Configuration.COMMAS) {

        @Override
        public int bufferSize() {
            // 300 is empirically measured to roughly produce ~20 chunks
            return 300;
        }
    };
    IdType idType = IdType.STRING;
    Decorator<InputNode> decorator = spy(new ExternalPropertiesDecorator(data(NO_NODE_DECORATOR, () -> decoratedData(count)), defaultFormatNodeFileHeader(), config, idType, UpdateBehaviour.ADD, collector));
    Input input = new CsvInput(nodeData(data(decorator, () -> mainData(count))), defaultFormatNodeFileHeader(), null, null, idType, config, collector, processors);
    // WHEN/THEN
    try (InputIterator<InputNode> nodes = input.nodes().iterator()) {
        int i = 0;
        for (; i < count; i++) {
            assertTrue(nodes.hasNext());
            InputNode node = nodes.next();
            // This property comes from decorator
            assertHasProperty(node, "extra", node.id() + "-decorated");
            if (i == 0) {
                // This code is equal to nodes.setProcessors( processors ) (a method which doesn't exist)
                nodes.processors(processors - nodes.processors(0));
            }
        }
        assertEquals(count, i);
        assertFalse(nodes.hasNext());
    }
    verify(decorator).close();
}

Example 3 with Input

use of org.neo4j.unsafe.impl.batchimport.input.Input in project neo4j by neo4j.

the class CsvInputTest method shouldProvideAdditiveLabels.

@Test
public void shouldProvideAdditiveLabels() throws Exception {
    // GIVEN
    String[] addedLabels = { "Two", "AddTwo" };
    DataFactory<InputNode> data = data(":ID,name,:LABEL\n" + "0,First,\n" + "1,Second,One\n" + "2,Third,One;Two", additiveLabels(addedLabels));
    Iterable<DataFactory<InputNode>> dataIterable = dataIterable(data);
    Input input = new CsvInput(dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.ACTUAL, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
    // WHEN/THEN
    try (ResourceIterator<InputNode> nodes = input.nodes().iterator()) {
        assertNode(nodes.next(), 0L, properties("name", "First"), labels(addedLabels));
        assertNode(nodes.next(), 1L, properties("name", "Second"), labels(union(new String[] { "One" }, addedLabels)));
        assertNode(nodes.next(), 2L, properties("name", "Third"), labels(union(new String[] { "One" }, addedLabels)));
        assertFalse(nodes.hasNext());
    }
}

Example 4 with Input

use of org.neo4j.unsafe.impl.batchimport.input.Input in project neo4j by neo4j.

the class CsvInputTest method shouldIgnoreEmptyPropertyValues.

@Test
public void shouldIgnoreEmptyPropertyValues() throws Exception {
    // GIVEN
    DataFactory<InputNode> data = data(":ID,name,extra\n" + // here we leave out "extra" property
    "0,Mattias,\n" + "1,Johan,Additional\n");
    Iterable<DataFactory<InputNode>> dataIterable = dataIterable(data);
    Input input = new CsvInput(dataIterable, defaultFormatNodeFileHeader(), null, null, IdType.ACTUAL, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
    // WHEN
    try (ResourceIterator<InputNode> nodes = input.nodes().iterator()) {
        // THEN
        assertNode(nodes.next(), 0L, new Object[] { "name", "Mattias" }, labels());
        assertNode(nodes.next(), 1L, new Object[] { "name", "Johan", "extra", "Additional" }, labels());
        assertFalse(nodes.hasNext());
    }
}

Example 5 with Input

use of org.neo4j.unsafe.impl.batchimport.input.Input in project neo4j by neo4j.

the class CsvInputTest method shouldDoWithoutRelationshipTypeHeaderIfDefaultSupplied.

@Test
public void shouldDoWithoutRelationshipTypeHeaderIfDefaultSupplied() throws Exception {
    // GIVEN relationship data w/o :TYPE header
    String defaultType = "HERE";
    DataFactory<InputRelationship> data = data(":START_ID,:END_ID,name\n" + "0,1,First\n" + "2,3,Second\n", defaultRelationshipType(defaultType));
    Iterable<DataFactory<InputRelationship>> dataIterable = dataIterable(data);
    Input input = new CsvInput(null, null, dataIterable, defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
    // WHEN
    try (ResourceIterator<InputRelationship> relationships = input.relationships().iterator()) {
        // THEN
        assertRelationship(relationships.next(), 0L, 1L, defaultType, properties("name", "First"));
        assertRelationship(relationships.next(), 2L, 3L, defaultType, properties("name", "Second"));
        assertFalse(relationships.hasNext());
    }
}