Examples with PropertyValueLookup org.neo4j.internal.batchimport.PropertyValueLookup used on opensource projects

Example 1 with PropertyValueLookup

use of org.neo4j.internal.batchimport.PropertyValueLookup in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups.

@Test
public void shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups() {
    // GIVEN
    final ControlledEncoder encoder = new ControlledEncoder(new LongEncoder());
    final int idsPerGroup = 20;
    int groupCount = 5;
    for (int i = 0; i < groupCount; i++) {
        groups.getOrCreate("Group " + i);
    }
    IdMapper mapper = mapper(encoder, Radix.LONG, EncodingIdMapper.NO_MONITOR, ParallelSort.DEFAULT, numberOfCollisions -> new LongCollisionValues(NumberArrayFactories.HEAP, numberOfCollisions, INSTANCE));
    final AtomicReference<Group> group = new AtomicReference<>();
    PropertyValueLookup ids = (nodeId, cursorContext) -> {
        int groupId = toIntExact(nodeId / idsPerGroup);
        if (groupId == groupCount) {
            return null;
        }
        group.set(groups.get(groupId));
        // i.e. all first 10% in each group collides with all other first 10% in each group
        if (nodeId % idsPerGroup < 2) {
            // Let these colliding values encode into the same eId as well,
            // so that they are definitely marked as collisions
            encoder.useThisIdToEncodeNoMatterWhatComesIn(1234567L);
            return nodeId % idsPerGroup;
        }
        // The other 90% will be accidental collisions for something else
        encoder.useThisIdToEncodeNoMatterWhatComesIn((long) (123456 - group.get().id()));
        return nodeId;
    };
    // WHEN
    int count = idsPerGroup * groupCount;
    for (long nodeId = 0; nodeId < count; nodeId++) {
        mapper.put(ids.lookupProperty(nodeId, NULL), nodeId, group.get());
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // THEN
    verifyNoMoreInteractions(collector);
    for (long nodeId = 0; nodeId < count; nodeId++) {
        assertEquals(nodeId, mapper.get(ids.lookupProperty(nodeId, NULL), group.get()));
    }
    verifyNoMoreInteractions(collector);
    assertFalse(mapper.leftOverDuplicateNodesIds().hasNext());
}

Example 2 with PropertyValueLookup

use of org.neo4j.internal.batchimport.PropertyValueLookup in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups.

@Test
public void shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups() {
    // GIVEN
    EncodingIdMapper.Monitor monitor = mock(EncodingIdMapper.Monitor.class);
    Group firstGroup = groups.getOrCreate("first");
    Group secondGroup = groups.getOrCreate("second");
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, monitor);
    PropertyValueLookup ids = values("10", "9", "10");
    int id = 0;
    // group 0
    mapper.put(ids.lookupProperty(id, NULL), id++, firstGroup);
    mapper.put(ids.lookupProperty(id, NULL), id++, firstGroup);
    // group 1
    mapper.put(ids.lookupProperty(id, NULL), id, secondGroup);
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // WHEN/THEN
    verifyNoMoreInteractions(collector);
    verify(monitor).numberOfCollisions(0);
    assertEquals(0L, mapper.get("10", firstGroup));
    assertEquals(1L, mapper.get("9", firstGroup));
    assertEquals(2L, mapper.get("10", secondGroup));
    assertFalse(mapper.leftOverDuplicateNodesIds().hasNext());
}

Example 3 with PropertyValueLookup

use of org.neo4j.internal.batchimport.PropertyValueLookup in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldReportCollisionsForSameInputId.

@Test
public void shouldReportCollisionsForSameInputId() {
    // GIVEN
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, EncodingIdMapper.NO_MONITOR);
    PropertyValueLookup values = values("10", "9", "10");
    for (int i = 0; i < 3; i++) {
        mapper.put(values.lookupProperty(i, NULL), i, Group.GLOBAL);
    }
    // WHEN
    Collector collector = mock(Collector.class);
    mapper.prepare(values, collector, NONE);
    // THEN
    verify(collector).collectDuplicateNode("10", 2, Group.GLOBAL.name());
    verifyNoMoreInteractions(collector);
}

Example 4 with PropertyValueLookup

use of org.neo4j.internal.batchimport.PropertyValueLookup in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldCopeWithCollisionsBasedOnDifferentInputIds.

@Test
public void shouldCopeWithCollisionsBasedOnDifferentInputIds() {
    // GIVEN
    EncodingIdMapper.Monitor monitor = mock(EncodingIdMapper.Monitor.class);
    Encoder encoder = mock(Encoder.class);
    when(encoder.encode(any())).thenReturn(12345L);
    IdMapper mapper = mapper(encoder, Radix.STRING, monitor);
    PropertyValueLookup ids = values("10", "9");
    for (int i = 0; i < 2; i++) {
        mapper.put(ids.lookupProperty(i, NULL), i, Group.GLOBAL);
    }
    // WHEN
    ProgressListener progress = mock(ProgressListener.class);
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, progress);
    // THEN
    verifyNoMoreInteractions(collector);
    verify(monitor).numberOfCollisions(2);
    assertEquals(0L, mapper.get("10", Group.GLOBAL));
    assertEquals(1L, mapper.get("9", Group.GLOBAL));
    // 7 times since SPLIT+SORT+DETECT+RESOLVE+SPLIT+SORT,DEDUPLICATE
    verify(progress, times(7)).started(anyString());
    verify(progress, times(7)).done();
}

Example 5 with PropertyValueLookup

use of org.neo4j.internal.batchimport.PropertyValueLookup in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldCopeWithMixedActualAndAccidentalCollisions.

@Test
public void shouldCopeWithMixedActualAndAccidentalCollisions() {
    // GIVEN
    EncodingIdMapper.Monitor monitor = mock(EncodingIdMapper.Monitor.class);
    Encoder encoder = mock(Encoder.class);
    // Create these explicit instances so that we can use them in mock, even for same values
    String a = "a";
    String b = "b";
    String c = "c";
    String a2 = "a";
    String e = "e";
    String f = "f";
    when(encoder.encode(a)).thenReturn(1L);
    when(encoder.encode(b)).thenReturn(1L);
    when(encoder.encode(c)).thenReturn(3L);
    when(encoder.encode(a2)).thenReturn(1L);
    when(encoder.encode(e)).thenReturn(2L);
    when(encoder.encode(f)).thenReturn(1L);
    Group groupA = groups.getOrCreate("A");
    Group groupB = groups.getOrCreate("B");
    IdMapper mapper = mapper(encoder, Radix.STRING, monitor);
    PropertyValueLookup ids = values("a", "b", "c", "a", "e", "f");
    Group[] groups = new Group[] { groupA, groupA, groupA, groupB, groupB, groupB };
    // WHEN
    for (int i = 0; i < 6; i++) {
        mapper.put(ids.lookupProperty(i, NULL), i, groups[i]);
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, mock(ProgressListener.class));
    // THEN
    verify(monitor).numberOfCollisions(4);
    assertEquals(0L, mapper.get(a, groupA));
    assertEquals(1L, mapper.get(b, groupA));
    assertEquals(2L, mapper.get(c, groupA));
    assertEquals(3L, mapper.get(a2, groupB));
    assertEquals(4L, mapper.get(e, groupB));
    assertEquals(5L, mapper.get(f, groupB));
}