Examples with Groups org.neo4j.unsafe.impl.batchimport.input.Groups used on opensource projects

Example 1 with Groups

use of org.neo4j.unsafe.impl.batchimport.input.Groups in project neo4j by neo4j.

the class InputGroupsDeserializerTest method shouldCoordinateGroupCreationForParallelProcessing.

@Test
public void shouldCoordinateGroupCreationForParallelProcessing() throws Exception {
    // GIVEN
    List<DataFactory<InputNode>> data = new ArrayList<>();
    int processors = Runtime.getRuntime().availableProcessors();
    for (int i = 0; i < processors; i++) {
        StringBuilder builder = new StringBuilder(":ID(Group" + i + ")");
        for (int j = 0; j < 100; j++) {
            builder.append("\n" + j);
        }
        data.add(data(builder.toString()));
    }
    Groups groups = new Groups();
    IdType idType = IdType.INTEGER;
    Collector badCollector = mock(Collector.class);
    Configuration config = lowBufferSize(COMMAS, false);
    DeserializerFactory<InputNode> factory = defaultNodeDeserializer(groups, config, idType, badCollector);
    try (InputGroupsDeserializer<InputNode> deserializer = new InputGroupsDeserializer<>(data.iterator(), defaultFormatNodeFileHeader(), config, idType, processors, processors, factory, Validators.<InputNode>emptyValidator(), InputNode.class)) {
        // WHEN
        count(deserializer);
    }
    // THEN
    assertEquals(processors, groups.getOrCreate("LastOne").id());
    boolean[] seen = new boolean[processors];
    for (int i = 0; i < processors; i++) {
        String groupName = "Group" + i;
        groups.getOrCreate(groupName);
        assertFalse(seen[i]);
        seen[i] = true;
    }
}

Example 2 with Groups

use of org.neo4j.unsafe.impl.batchimport.input.Groups in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups.

@Test
public void shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups() throws Exception {
    // GIVEN
    Monitor monitor = mock(Monitor.class);
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, monitor);
    InputIterable<Object> ids = wrap("source", Arrays.<Object>asList("10", "9", "10"));
    Groups groups = new Groups();
    Group firstGroup = groups.getOrCreate("first"), secondGroup = groups.getOrCreate("second");
    try (ResourceIterator<Object> iterator = ids.iterator()) {
        int id = 0;
        // group 0
        mapper.put(iterator.next(), id++, firstGroup);
        mapper.put(iterator.next(), id++, firstGroup);
        // group 1
        mapper.put(iterator.next(), id++, secondGroup);
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // WHEN/THEN
    verifyNoMoreInteractions(collector);
    verify(monitor).numberOfCollisions(0);
    assertEquals(0L, mapper.get("10", firstGroup));
    assertEquals(1L, mapper.get("9", firstGroup));
    assertEquals(2L, mapper.get("10", secondGroup));
}

Example 3 with Groups

use of org.neo4j.unsafe.impl.batchimport.input.Groups in project neo4j by neo4j.

the class CsvInputTest method shouldHaveNodesBelongToGroupSpecifiedInHeader.

@Test
public void shouldHaveNodesBelongToGroupSpecifiedInHeader() throws Exception {
    // GIVEN
    IdType idType = IdType.ACTUAL;
    Iterable<DataFactory<InputNode>> data = dataIterable(data("123,one\n" + "456,two"));
    Groups groups = new Groups();
    Group group = groups.getOrCreate("MyGroup");
    Input input = new CsvInput(data, header(entry(null, Type.ID, group.name(), idType.extractor(extractors)), entry("name", Type.PROPERTY, extractors.string())), null, null, idType, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
    // WHEN/THEN
    try (InputIterator<InputNode> nodes = input.nodes().iterator()) {
        assertNode(nodes.next(), group, 123L, properties("name", "one"), labels());
        assertNode(nodes.next(), group, 456L, properties("name", "two"), labels());
        assertFalse(nodes.hasNext());
    }
}

Example 4 with Groups

use of org.neo4j.unsafe.impl.batchimport.input.Groups in project neo4j by neo4j.

the class CsvInputTest method shouldHaveRelationshipsSpecifyStartEndNodeIdGroupsInHeader.

@Test
public void shouldHaveRelationshipsSpecifyStartEndNodeIdGroupsInHeader() throws Exception {
    // GIVEN
    IdType idType = IdType.ACTUAL;
    Iterable<DataFactory<InputRelationship>> data = dataIterable(data("123,TYPE,234\n" + "345,TYPE,456"));
    Groups groups = new Groups();
    Group startNodeGroup = groups.getOrCreate("StartGroup");
    Group endNodeGroup = groups.getOrCreate("EndGroup");
    Input input = new CsvInput(null, null, data, header(entry(null, Type.START_ID, startNodeGroup.name(), idType.extractor(extractors)), entry(null, Type.TYPE, extractors.string()), entry(null, Type.END_ID, endNodeGroup.name(), idType.extractor(extractors))), idType, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
    // WHEN/THEN
    try (InputIterator<InputRelationship> relationships = input.relationships().iterator()) {
        assertRelationship(relationships.next(), startNodeGroup, 123L, endNodeGroup, 234L, "TYPE", properties());
        assertRelationship(relationships.next(), startNodeGroup, 345L, endNodeGroup, 456L, "TYPE", properties());
        assertFalse(relationships.hasNext());
    }
}

Example 5 with Groups

use of org.neo4j.unsafe.impl.batchimport.input.Groups in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldOnlyFindInputIdsInSpecificGroup.

@Test
public void shouldOnlyFindInputIdsInSpecificGroup() throws Exception {
    // GIVEN
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, NO_MONITOR);
    InputIterable<Object> ids = wrap("source", Arrays.<Object>asList("8", "9", "10"));
    Groups groups = new Groups();
    Group firstGroup, secondGroup, thirdGroup;
    try (ResourceIterator<Object> iterator = ids.iterator()) {
        int id = 0;
        mapper.put(iterator.next(), id++, firstGroup = groups.getOrCreate("first"));
        mapper.put(iterator.next(), id++, secondGroup = groups.getOrCreate("second"));
        mapper.put(iterator.next(), id++, thirdGroup = groups.getOrCreate("third"));
    }
    mapper.prepare(ids, mock(Collector.class), NONE);
    // WHEN/THEN
    assertEquals(0L, mapper.get("8", firstGroup));
    assertEquals(ID_NOT_FOUND, mapper.get("8", secondGroup));
    assertEquals(ID_NOT_FOUND, mapper.get("8", thirdGroup));
    assertEquals(ID_NOT_FOUND, mapper.get("9", firstGroup));
    assertEquals(1L, mapper.get("9", secondGroup));
    assertEquals(ID_NOT_FOUND, mapper.get("9", thirdGroup));
    assertEquals(ID_NOT_FOUND, mapper.get("10", firstGroup));
    assertEquals(ID_NOT_FOUND, mapper.get("10", secondGroup));
    assertEquals(2L, mapper.get("10", thirdGroup));
}