Search in sources :

Example 1 with DeserializerFactory

use of org.neo4j.unsafe.impl.batchimport.input.csv.InputGroupsDeserializer.DeserializerFactory in project neo4j by neo4j.

the class ParallelInputEntityDeserializerTest method shouldParseDataInParallel.

@Test
public void shouldParseDataInParallel() throws Exception {
    // GIVEN
    int entities = 500;
    Data<InputNode> data = testData(entities);
    Configuration config = new Configuration.Overridden(COMMAS) {

        @Override
        public int bufferSize() {
            return 100;
        }
    };
    IdType idType = ACTUAL;
    Collector badCollector = mock(Collector.class);
    Groups groups = new Groups();
    Set<Thread> observedProcessingThreads = new CopyOnWriteArraySet<>();
    int threads = 4;
    DeserializerFactory<InputNode> deserializerFactory = (header, chunk, decorator, validator) -> {
        observedProcessingThreads.add(Thread.currentThread());
        // Make sure there will be 4 different processing threads doing this
        boolean allThreadsStarted;
        do {
            allThreadsStarted = observedProcessingThreads.size() == threads;
        } while (!allThreadsStarted);
        return new InputEntityDeserializer<>(header, chunk, config.delimiter(), new InputNodeDeserialization(header, chunk, groups, idType.idsAreExternal()), decorator, validator, badCollector);
    };
    try (ParallelInputEntityDeserializer<InputNode> deserializer = new ParallelInputEntityDeserializer<>(data, defaultFormatNodeFileHeader(), config, idType, threads, threads, deserializerFactory, Validators.<InputNode>emptyValidator(), InputNode.class)) {
        // WHEN/THEN
        long previousLineNumber = -1;
        long previousPosition = -1;
        for (long i = 0; i < entities; i++) {
            assertTrue(deserializer.hasNext());
            InputNode entity = deserializer.next();
            assertEquals(i, ((Long) entity.id()).longValue());
            assertEquals("name", entity.properties()[0]);
            assertTrue(entity.properties()[1].toString().startsWith(i + "-"));
            assertTrue(entity.lineNumber() > previousLineNumber);
            previousLineNumber = entity.lineNumber();
            assertTrue(entity.position() > previousPosition);
            previousPosition = entity.position();
        }
        assertFalse(deserializer.hasNext());
        assertEquals(threads, observedProcessingThreads.size());
    }
}
Also used : InputNode(org.neo4j.unsafe.impl.batchimport.input.InputNode) Validators(org.neo4j.kernel.impl.util.Validators) TaskExecutionPanicException(org.neo4j.unsafe.impl.batchimport.executor.TaskExecutionPanicException) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) ACTUAL(org.neo4j.unsafe.impl.batchimport.input.csv.IdType.ACTUAL) CopyOnWriteArraySet(java.util.concurrent.CopyOnWriteArraySet) DeserializerFactories.defaultNodeDeserializer(org.neo4j.unsafe.impl.batchimport.input.csv.DeserializerFactories.defaultNodeDeserializer) CharReadable(org.neo4j.csv.reader.CharReadable) DataFactories.defaultFormatNodeFileHeader(org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatNodeFileHeader) Groups(org.neo4j.unsafe.impl.batchimport.input.Groups) Rule(org.junit.Rule) Readables.wrap(org.neo4j.csv.reader.Readables.wrap) StringReader(java.io.StringReader) RandomRule(org.neo4j.test.rule.RandomRule) Assert.assertFalse(org.junit.Assert.assertFalse) Collector(org.neo4j.unsafe.impl.batchimport.input.Collector) DeserializerFactory(org.neo4j.unsafe.impl.batchimport.input.csv.InputGroupsDeserializer.DeserializerFactory) InputEntityDecorators(org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators) COMMAS(org.neo4j.unsafe.impl.batchimport.input.csv.Configuration.COMMAS) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) InputNode(org.neo4j.unsafe.impl.batchimport.input.InputNode) CopyOnWriteArraySet(java.util.concurrent.CopyOnWriteArraySet) Groups(org.neo4j.unsafe.impl.batchimport.input.Groups) Collector(org.neo4j.unsafe.impl.batchimport.input.Collector) Test(org.junit.Test)

Aggregations

StringReader (java.io.StringReader)1 Set (java.util.Set)1 CopyOnWriteArraySet (java.util.concurrent.CopyOnWriteArraySet)1 Assert.assertEquals (org.junit.Assert.assertEquals)1 Assert.assertFalse (org.junit.Assert.assertFalse)1 Assert.assertTrue (org.junit.Assert.assertTrue)1 Rule (org.junit.Rule)1 Test (org.junit.Test)1 Mockito.mock (org.mockito.Mockito.mock)1 CharReadable (org.neo4j.csv.reader.CharReadable)1 Readables.wrap (org.neo4j.csv.reader.Readables.wrap)1 Validators (org.neo4j.kernel.impl.util.Validators)1 RandomRule (org.neo4j.test.rule.RandomRule)1 TaskExecutionPanicException (org.neo4j.unsafe.impl.batchimport.executor.TaskExecutionPanicException)1 Collector (org.neo4j.unsafe.impl.batchimport.input.Collector)1 Groups (org.neo4j.unsafe.impl.batchimport.input.Groups)1 InputEntityDecorators (org.neo4j.unsafe.impl.batchimport.input.InputEntityDecorators)1 InputNode (org.neo4j.unsafe.impl.batchimport.input.InputNode)1 COMMAS (org.neo4j.unsafe.impl.batchimport.input.csv.Configuration.COMMAS)1 DataFactories.defaultFormatNodeFileHeader (org.neo4j.unsafe.impl.batchimport.input.csv.DataFactories.defaultFormatNodeFileHeader)1