Examples with Configuration org.neo4j.unsafe.impl.batchimport.input.csv.Configuration used on opensource projects

Example 1 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportTool method csvConfiguration.

public static Configuration csvConfiguration(Args args, final boolean defaultSettingsSuitableForTests) {
    final Configuration defaultConfiguration = COMMAS;
    final Character specificDelimiter = args.interpretOption(Options.DELIMITER.key(), Converters.<Character>optional(), CHARACTER_CONVERTER);
    final Character specificArrayDelimiter = args.interpretOption(Options.ARRAY_DELIMITER.key(), Converters.<Character>optional(), CHARACTER_CONVERTER);
    final Character specificQuote = args.interpretOption(Options.QUOTE.key(), Converters.<Character>optional(), CHARACTER_CONVERTER);
    final Boolean multiLineFields = args.getBoolean(Options.MULTILINE_FIELDS.key(), null);
    final Boolean emptyStringsAsNull = args.getBoolean(Options.IGNORE_EMPTY_STRINGS.key(), null);
    final Boolean trimStrings = args.getBoolean(Options.TRIM_STRINGS.key(), null);
    final Boolean legacyStyleQuoting = args.getBoolean(Options.LEGACY_STYLE_QUOTING.key(), null);
    return new Configuration.Default() {

        @Override
        public char delimiter() {
            return specificDelimiter != null ? specificDelimiter.charValue() : defaultConfiguration.delimiter();
        }

        @Override
        public char arrayDelimiter() {
            return specificArrayDelimiter != null ? specificArrayDelimiter.charValue() : defaultConfiguration.arrayDelimiter();
        }

        @Override
        public char quotationCharacter() {
            return specificQuote != null ? specificQuote.charValue() : defaultConfiguration.quotationCharacter();
        }

        @Override
        public boolean multilineFields() {
            return multiLineFields != null ? multiLineFields.booleanValue() : defaultConfiguration.multilineFields();
        }

        @Override
        public boolean emptyQuotedStringsAsNull() {
            return emptyStringsAsNull != null ? emptyStringsAsNull.booleanValue() : defaultConfiguration.emptyQuotedStringsAsNull();
        }

        @Override
        public int bufferSize() {
            return defaultSettingsSuitableForTests ? 10_000 : super.bufferSize();
        }

        @Override
        public boolean trimStrings() {
            return trimStrings != null ? trimStrings.booleanValue() : defaultConfiguration.trimStrings();
        }

        @Override
        public boolean legacyStyleQuoting() {
            return legacyStyleQuoting != null ? legacyStyleQuoting.booleanValue() : defaultConfiguration.legacyStyleQuoting();
        }
    };
}

Example 2 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldLogRelationshipsReferringToMissingNode.

@Test
public void shouldLogRelationshipsReferringToMissingNode() throws Exception {
    // GIVEN
    List<String> nodeIds = asList("a", "b", "c");
    Configuration config = Configuration.COMMAS;
    File nodeData = nodeData(true, config, nodeIds, TRUE);
    List<RelationshipDataLine> relationships = Arrays.asList(//          line 2 of file1
    relationship("a", "b", "TYPE", "aa"), //      line 3 of file1
    relationship("c", "bogus", "TYPE", "bb"), //         line 1 of file2
    relationship("b", "c", "KNOWS", "cc"), //         line 2 of file2
    relationship("c", "a", "KNOWS", "dd"), // line 3 of file2
    relationship("missing", "a", "KNOWS", "ee"));
    File relationshipData1 = relationshipData(true, config, relationships.iterator(), lines(0, 2), true);
    File relationshipData2 = relationshipData(false, config, relationships.iterator(), lines(2, 5), true);
    File bad = badFile();
    // WHEN importing data where some relationships refer to missing nodes
    importTool("--into", dbRule.getStoreDirAbsolutePath(), "--nodes", nodeData.getAbsolutePath(), "--bad", bad.getAbsolutePath(), "--bad-tolerance", "2", "--relationships", relationshipData1.getAbsolutePath() + MULTI_FILE_DELIMITER + relationshipData2.getAbsolutePath());
    // THEN
    String badContents = FileUtils.readTextFile(bad, Charset.defaultCharset());
    assertTrue("Didn't contain first bad relationship", badContents.contains(relationshipData1.getAbsolutePath() + ":3"));
    assertTrue("Didn't contain second bad relationship", badContents.contains(relationshipData2.getAbsolutePath() + ":3"));
    verifyRelationships(relationships);
}

Example 3 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldPrintStackTraceOnInputExceptionIfToldTo.

@Test
public void shouldPrintStackTraceOnInputExceptionIfToldTo() throws Exception {
    // GIVEN
    List<String> nodeIds = nodeIds();
    Configuration config = Configuration.TABS;
    // WHEN data file contains more columns than header file
    int extraColumns = 3;
    try {
        importTool("--into", dbRule.getStoreDirAbsolutePath(), "--nodes", nodeHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, nodeIds, TRUE, Charset.defaultCharset(), extraColumns).getAbsolutePath(), "--stacktrace");
        fail("Should have thrown exception");
    } catch (InputException e) {
        // THEN
        assertTrue(suppressOutput.getErrorVoice().containsMessage(e.getClass().getName()));
        assertTrue(e.getMessage().contains("Extra column not present in header on line"));
    }
}

Example 4 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldImportWithoutTypeSpecifiedInRelationshipHeaderbutWithDefaultTypeInArgument.

@Test
public void shouldImportWithoutTypeSpecifiedInRelationshipHeaderbutWithDefaultTypeInArgument() throws Exception {
    // GIVEN
    List<String> nodeIds = nodeIds();
    Configuration config = Configuration.COMMAS;
    String type = randomType();
    // WHEN
    importTool("--into", dbRule.getStoreDirAbsolutePath(), "--nodes", nodeData(true, config, nodeIds, TRUE).getAbsolutePath(), // there will be no :TYPE specified in the header of the relationships below
    "--relationships:" + type, relationshipData(true, config, nodeIds, TRUE, false).getAbsolutePath());
    // THEN
    verifyData();
}

Example 5 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldPrintReferenceLinkAsPartOfErrorMessage.

private void shouldPrintReferenceLinkAsPartOfErrorMessage(List<String> nodeIds, Iterator<RelationshipDataLine> relationshipDataLines, String message) throws Exception {
    Configuration config = Configuration.COMMAS;
    try {
        // WHEN
        importTool("--into", dbRule.getStoreDirAbsolutePath(), "--nodes", nodeData(true, config, nodeIds, TRUE).getAbsolutePath(), "--relationships", relationshipData(true, config, relationshipDataLines, TRUE, true).getAbsolutePath());
        fail(" Should fail during import.");
    } catch (Exception e) {
        // EXPECT
        assertTrue(suppressOutput.getErrorVoice().containsMessage(message));
    }
}