Examples with IdMapper org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper used on opensource projects

Example 1 with IdMapper

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldHandleGreatAmountsOfStuff.

@Test
public void shouldHandleGreatAmountsOfStuff() throws Exception {
    // GIVEN
    IdMapper idMapper = mapper(new StringEncoder(), Radix.STRING, NO_MONITOR);
    InputIterable<Object> ids = new InputIterable<Object>() {

        @Override
        public InputIterator<Object> iterator() {
            return new InputIterator.Adapter<Object>() {

                private int i;

                @Override
                protected Object fetchNextOrNull() {
                    return i < 300_000 ? "" + (i++) : null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return false;
        }
    };
    // WHEN
    long index = 0;
    for (Object id : ids) {
        idMapper.put(id, index++, GLOBAL);
    }
    idMapper.prepare(ids, mock(Collector.class), NONE);
    // THEN
    for (Object id : ids) {
        // the UUIDs here will be generated in the same sequence as above because we reset the random
        if (idMapper.get(id, GLOBAL) == ID_NOT_FOUND) {
            fail("Couldn't find " + id + " even though I added it just previously");
        }
    }
}

Example 2 with IdMapper

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldHandleManyGroups.

@Test
public void shouldHandleManyGroups() throws Exception {
    // GIVEN
    IdMapper mapper = mapper(new LongEncoder(), Radix.LONG, NO_MONITOR);
    int size = 100;
    // WHEN
    for (int i = 0; i < size; i++) {
        mapper.put(i, i, new Group.Adapter(i, "" + i));
    }
    // null since this test should have been set up to not run into collisions
    mapper.prepare(null, mock(Collector.class), NONE);
    // THEN
    for (int i = 0; i < size; i++) {
        assertEquals(i, mapper.get(i, new Group.Adapter(i, "" + i)));
    }
}

Example 3 with IdMapper

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups.

@Test
public void shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups() throws Exception {
    // GIVEN
    Monitor monitor = mock(Monitor.class);
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, monitor);
    InputIterable<Object> ids = wrap("source", Arrays.<Object>asList("10", "9", "10"));
    Groups groups = new Groups();
    Group firstGroup = groups.getOrCreate("first"), secondGroup = groups.getOrCreate("second");
    try (ResourceIterator<Object> iterator = ids.iterator()) {
        int id = 0;
        // group 0
        mapper.put(iterator.next(), id++, firstGroup);
        mapper.put(iterator.next(), id++, firstGroup);
        // group 1
        mapper.put(iterator.next(), id++, secondGroup);
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // WHEN/THEN
    verifyNoMoreInteractions(collector);
    verify(monitor).numberOfCollisions(0);
    assertEquals(0L, mapper.get("10", firstGroup));
    assertEquals(1L, mapper.get("9", firstGroup));
    assertEquals(2L, mapper.get("10", secondGroup));
}

Example 4 with IdMapper

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups.

@Test
public void shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups() throws Exception {
    // GIVEN
    final ControlledEncoder encoder = new ControlledEncoder(new LongEncoder());
    IdMapper mapper = mapper(encoder, Radix.LONG, NO_MONITOR);
    final int idsPerGroup = 20, groups = 5;
    final AtomicReference<Group> group = new AtomicReference<>();
    InputIterable<Object> ids = SimpleInputIteratorWrapper.wrap("source", new Iterable<Object>() {

        @Override
        public Iterator<Object> iterator() {
            return new PrefetchingIterator<Object>() {

                private int i;

                @Override
                protected Object fetchNextOrNull() {
                    // Change group every <idsPerGroup> id
                    if (i % idsPerGroup == 0) {
                        int groupId = i / idsPerGroup;
                        if (groupId == groups) {
                            return null;
                        }
                        group.set(new Group.Adapter(groupId, "Group " + groupId));
                    }
                    try {
                        // i.e. all first 10% in each group collides with all other first 10% in each group
                        if (i % idsPerGroup < 2) {
                            // Let these colliding values encode into the same eId as well,
                            // so that they are definitely marked as collisions
                            encoder.useThisIdToEncodeNoMatterWhatComesIn(Long.valueOf(1234567));
                            return Long.valueOf(i % idsPerGroup);
                        }
                        // The other 90% will be accidental collisions for something else
                        encoder.useThisIdToEncodeNoMatterWhatComesIn(Long.valueOf(123456 - group.get().id()));
                        return Long.valueOf(i);
                    } finally {
                        i++;
                    }
                }
            };
        }
    });
    // WHEN
    long actualId = 0;
    for (Object id : ids) {
        mapper.put(id, actualId++, group.get());
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // THEN
    verifyNoMoreInteractions(collector);
    actualId = 0;
    for (Object id : ids) {
        assertEquals(actualId++, mapper.get(id, group.get()));
    }
}

Example 5 with IdMapper

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldDetectLargeAmountsOfCollisions.

@Test
public void shouldDetectLargeAmountsOfCollisions() throws Exception {
    // GIVEN
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, NO_MONITOR);
    int count = EncodingIdMapper.COUNTING_BATCH_SIZE * 2;
    List<Object> ids = new ArrayList<>();
    long id = 0;
    // Generate and add all input ids
    while (id < count) {
        String inputId = UUID.randomUUID().toString();
        ids.add(inputId);
        mapper.put(inputId, id++, GLOBAL);
    }
    // And add them one more time
    for (Object inputId : ids) {
        mapper.put(inputId, id++, GLOBAL);
    }
    ids.addAll(ids);
    // WHEN
    CountingCollector collector = new CountingCollector();
    mapper.prepare(SimpleInputIteratorWrapper.wrap("source", ids), collector, NONE);
    // THEN
    assertEquals(count, collector.count);
}