Examples with Configuration org.neo4j.unsafe.impl.batchimport.input.csv.Configuration used on opensource projects

Example 16 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldNotBeAbleToMixSpecifiedAndUnspecifiedGroups.

@Test
public void shouldNotBeAbleToMixSpecifiedAndUnspecifiedGroups() throws Exception {
    // GIVEN
    List<String> groupOneNodeIds = asList("1", "2", "3");
    List<String> groupTwoNodeIds = asList("4", "5", "2");
    Configuration config = Configuration.COMMAS;
    // WHEN
    try {
        importTool("--into", dbRule.getStoreDirAbsolutePath(), "--nodes", nodeHeader(config, "MyGroup").getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, groupOneNodeIds, TRUE).getAbsolutePath(), "--nodes", nodeHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, groupTwoNodeIds, TRUE).getAbsolutePath());
        fail("Should have failed");
    } catch (Exception e) {
        assertExceptionContains(e, "Mixing specified", IllegalStateException.class);
    }
}

Example 17 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldImportSplitInputFiles.

@Test
public void shouldImportSplitInputFiles() throws Exception {
    // GIVEN
    List<String> nodeIds = nodeIds();
    Configuration config = Configuration.COMMAS;
    // WHEN
    importTool("--into", dbRule.getStoreDirAbsolutePath(), // One group with one header file and one data file
    "--nodes", nodeHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, nodeIds, lines(0, NODE_COUNT / 2)).getAbsolutePath(), // One group with two data files, where the header sits in the first file
    "--nodes", nodeData(true, config, nodeIds, lines(NODE_COUNT / 2, NODE_COUNT * 3 / 4)).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, nodeIds, lines(NODE_COUNT * 3 / 4, NODE_COUNT)).getAbsolutePath(), "--relationships", relationshipHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + relationshipData(false, config, nodeIds, TRUE, true).getAbsolutePath());
    // THEN
    verifyData();
}

Example 18 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class QuickImport method main.

public static void main(String[] arguments) throws IOException {
    Args args = Args.parse(arguments);
    long nodeCount = parseLongWithUnit(args.get("nodes", null));
    long relationshipCount = parseLongWithUnit(args.get("relationships", null));
    int labelCount = args.getNumber("labels", 4).intValue();
    int relationshipTypeCount = args.getNumber("relationship-types", 4).intValue();
    File dir = new File(args.get(ImportTool.Options.STORE_DIR.key()));
    long randomSeed = args.getNumber("random-seed", currentTimeMillis()).longValue();
    Configuration config = COMMAS;
    Extractors extractors = new Extractors(config.arrayDelimiter());
    IdType idType = IdType.valueOf(args.get("id-type", IdType.ACTUAL.name()));
    Header nodeHeader = parseNodeHeader(args, idType, extractors);
    Header relationshipHeader = parseRelationshipHeader(args, idType, extractors);
    FormattedLogProvider sysoutLogProvider = FormattedLogProvider.toOutputStream(System.out);
    org.neo4j.unsafe.impl.batchimport.Configuration importConfig = new Default() {

        @Override
        public int maxNumberOfProcessors() {
            return args.getNumber(ImportTool.Options.PROCESSORS.key(), super.maxNumberOfProcessors()).intValue();
        }

        @Override
        public int denseNodeThreshold() {
            return args.getNumber(dense_node_threshold.name(), super.denseNodeThreshold()).intValue();
        }
    };
    SimpleDataGenerator generator = new SimpleDataGenerator(nodeHeader, relationshipHeader, randomSeed, nodeCount, labelCount, relationshipTypeCount, idType);
    Input input = new DataGeneratorInput(nodeCount, relationshipCount, generator.nodes(), generator.relationships(), idType, silentBadCollector(0));
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction()) {
        BatchImporter consumer;
        if (args.getBoolean("to-csv")) {
            consumer = new CsvOutput(dir, nodeHeader, relationshipHeader, config);
        } else {
            consumer = new ParallelBatchImporter(dir, fileSystem, importConfig, new SimpleLogService(sysoutLogProvider, sysoutLogProvider), defaultVisible(), Config.defaults());
        }
        consumer.doImport(input);
    }
}

Example 19 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class QuickImport method parseRelationshipHeader.

private static Header parseRelationshipHeader(Args args, IdType idType, Extractors extractors) {
    String definition = args.get("relationship-header", null);
    if (definition == null) {
        return bareboneRelationshipHeader(idType, extractors);
    }
    Configuration config = Configuration.COMMAS;
    return defaultFormatRelationshipFileHeader().create(seeker(definition, config), config, idType);
}

Example 20 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class QuickImport method parseNodeHeader.

private static Header parseNodeHeader(Args args, IdType idType, Extractors extractors) {
    String definition = args.get("node-header", null);
    if (definition == null) {
        return bareboneNodeHeader(idType, extractors);
    }
    Configuration config = Configuration.COMMAS;
    return defaultFormatNodeFileHeader().create(seeker(definition, config), config, idType);
}