Search in sources :

Example 16 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldNotBeAbleToMixSpecifiedAndUnspecifiedGroups.

@Test
public void shouldNotBeAbleToMixSpecifiedAndUnspecifiedGroups() throws Exception {
    // GIVEN
    List<String> groupOneNodeIds = asList("1", "2", "3");
    List<String> groupTwoNodeIds = asList("4", "5", "2");
    Configuration config = Configuration.COMMAS;
    // WHEN
    try {
        importTool("--into", dbRule.getStoreDirAbsolutePath(), "--nodes", nodeHeader(config, "MyGroup").getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, groupOneNodeIds, TRUE).getAbsolutePath(), "--nodes", nodeHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, groupTwoNodeIds, TRUE).getAbsolutePath());
        fail("Should have failed");
    } catch (Exception e) {
        assertExceptionContains(e, "Mixing specified", IllegalStateException.class);
    }
}
Also used : Configuration(org.neo4j.unsafe.impl.batchimport.input.csv.Configuration) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) IllegalMultilineFieldException(org.neo4j.csv.reader.IllegalMultilineFieldException) FileNotFoundException(java.io.FileNotFoundException) DuplicateInputIdException(org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.DuplicateInputIdException) InputException(org.neo4j.unsafe.impl.batchimport.input.InputException) IOException(java.io.IOException) Test(org.junit.Test)

Example 17 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class ImportToolTest method shouldImportSplitInputFiles.

@Test
public void shouldImportSplitInputFiles() throws Exception {
    // GIVEN
    List<String> nodeIds = nodeIds();
    Configuration config = Configuration.COMMAS;
    // WHEN
    importTool("--into", dbRule.getStoreDirAbsolutePath(), // One group with one header file and one data file
    "--nodes", nodeHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, nodeIds, lines(0, NODE_COUNT / 2)).getAbsolutePath(), // One group with two data files, where the header sits in the first file
    "--nodes", nodeData(true, config, nodeIds, lines(NODE_COUNT / 2, NODE_COUNT * 3 / 4)).getAbsolutePath() + MULTI_FILE_DELIMITER + nodeData(false, config, nodeIds, lines(NODE_COUNT * 3 / 4, NODE_COUNT)).getAbsolutePath(), "--relationships", relationshipHeader(config).getAbsolutePath() + MULTI_FILE_DELIMITER + relationshipData(false, config, nodeIds, TRUE, true).getAbsolutePath());
    // THEN
    verifyData();
}
Also used : Configuration(org.neo4j.unsafe.impl.batchimport.input.csv.Configuration) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Example 18 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class QuickImport method main.

public static void main(String[] arguments) throws IOException {
    Args args = Args.parse(arguments);
    long nodeCount = parseLongWithUnit(args.get("nodes", null));
    long relationshipCount = parseLongWithUnit(args.get("relationships", null));
    int labelCount = args.getNumber("labels", 4).intValue();
    int relationshipTypeCount = args.getNumber("relationship-types", 4).intValue();
    File dir = new File(args.get(ImportTool.Options.STORE_DIR.key()));
    long randomSeed = args.getNumber("random-seed", currentTimeMillis()).longValue();
    Configuration config = COMMAS;
    Extractors extractors = new Extractors(config.arrayDelimiter());
    IdType idType = IdType.valueOf(args.get("id-type", IdType.ACTUAL.name()));
    Header nodeHeader = parseNodeHeader(args, idType, extractors);
    Header relationshipHeader = parseRelationshipHeader(args, idType, extractors);
    FormattedLogProvider sysoutLogProvider = FormattedLogProvider.toOutputStream(System.out);
    org.neo4j.unsafe.impl.batchimport.Configuration importConfig = new Default() {

        @Override
        public int maxNumberOfProcessors() {
            return args.getNumber(ImportTool.Options.PROCESSORS.key(), super.maxNumberOfProcessors()).intValue();
        }

        @Override
        public int denseNodeThreshold() {
            return args.getNumber(dense_node_threshold.name(), super.denseNodeThreshold()).intValue();
        }
    };
    SimpleDataGenerator generator = new SimpleDataGenerator(nodeHeader, relationshipHeader, randomSeed, nodeCount, labelCount, relationshipTypeCount, idType);
    Input input = new DataGeneratorInput(nodeCount, relationshipCount, generator.nodes(), generator.relationships(), idType, silentBadCollector(0));
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction()) {
        BatchImporter consumer;
        if (args.getBoolean("to-csv")) {
            consumer = new CsvOutput(dir, nodeHeader, relationshipHeader, config);
        } else {
            consumer = new ParallelBatchImporter(dir, fileSystem, importConfig, new SimpleLogService(sysoutLogProvider, sysoutLogProvider), defaultVisible(), Config.defaults());
        }
        consumer.doImport(input);
    }
}
Also used : Args(org.neo4j.helpers.Args) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) Configuration(org.neo4j.unsafe.impl.batchimport.input.csv.Configuration) SimpleLogService(org.neo4j.kernel.impl.logging.SimpleLogService) Default(org.neo4j.unsafe.impl.batchimport.Configuration.Default) IdType(org.neo4j.unsafe.impl.batchimport.input.csv.IdType) FormattedLogProvider(org.neo4j.logging.FormattedLogProvider) ParallelBatchImporter(org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter) Extractors(org.neo4j.csv.reader.Extractors) Input(org.neo4j.unsafe.impl.batchimport.input.Input) Header(org.neo4j.unsafe.impl.batchimport.input.csv.Header) DataGeneratorInput.bareboneNodeHeader(org.neo4j.tooling.DataGeneratorInput.bareboneNodeHeader) DataFactories.defaultFormatRelationshipFileHeader(org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatRelationshipFileHeader) DataFactories.defaultFormatNodeFileHeader(org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatNodeFileHeader) DataGeneratorInput.bareboneRelationshipHeader(org.neo4j.tooling.DataGeneratorInput.bareboneRelationshipHeader) ParallelBatchImporter(org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter) BatchImporter(org.neo4j.unsafe.impl.batchimport.BatchImporter) File(java.io.File)

Example 19 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class QuickImport method parseRelationshipHeader.

private static Header parseRelationshipHeader(Args args, IdType idType, Extractors extractors) {
    String definition = args.get("relationship-header", null);
    if (definition == null) {
        return bareboneRelationshipHeader(idType, extractors);
    }
    Configuration config = Configuration.COMMAS;
    return defaultFormatRelationshipFileHeader().create(seeker(definition, config), config, idType);
}
Also used : Configuration(org.neo4j.unsafe.impl.batchimport.input.csv.Configuration)

Example 20 with Configuration

use of org.neo4j.unsafe.impl.batchimport.input.csv.Configuration in project neo4j by neo4j.

the class QuickImport method parseNodeHeader.

private static Header parseNodeHeader(Args args, IdType idType, Extractors extractors) {
    String definition = args.get("node-header", null);
    if (definition == null) {
        return bareboneNodeHeader(idType, extractors);
    }
    Configuration config = Configuration.COMMAS;
    return defaultFormatNodeFileHeader().create(seeker(definition, config), config, idType);
}
Also used : Configuration(org.neo4j.unsafe.impl.batchimport.input.csv.Configuration)

Aggregations

Configuration (org.neo4j.unsafe.impl.batchimport.input.csv.Configuration)27 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)22 Test (org.junit.Test)22 File (java.io.File)7 InputException (org.neo4j.unsafe.impl.batchimport.input.InputException)7 FileNotFoundException (java.io.FileNotFoundException)5 IOException (java.io.IOException)5 IllegalMultilineFieldException (org.neo4j.csv.reader.IllegalMultilineFieldException)5 DuplicateInputIdException (org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.DuplicateInputIdException)5 GraphDatabaseService (org.neo4j.graphdb.GraphDatabaseService)4 Node (org.neo4j.graphdb.Node)4 Transaction (org.neo4j.graphdb.Transaction)4 DynamicLabel (org.neo4j.graphdb.DynamicLabel)2 Label (org.neo4j.graphdb.Label)2 Charset (java.nio.charset.Charset)1 Extractors (org.neo4j.csv.reader.Extractors)1 Args (org.neo4j.helpers.Args)1 DefaultFileSystemAbstraction (org.neo4j.io.fs.DefaultFileSystemAbstraction)1 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)1 SimpleLogService (org.neo4j.kernel.impl.logging.SimpleLogService)1