Examples with IdMapper org.neo4j.internal.batchimport.cache.idmapping.IdMapper used on opensource projects

Example 1 with IdMapper

use of org.neo4j.internal.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class NodeImporterTest method shouldHandleLargeAmountsOfLabels.

@Test
void shouldHandleLargeAmountsOfLabels() throws IOException {
    // given
    IdMapper idMapper = mock(IdMapper.class);
    JobScheduler scheduler = new ThreadPoolJobScheduler();
    try (Lifespan life = new Lifespan(scheduler);
        BatchingNeoStores stores = BatchingNeoStores.batchingNeoStoresWithExternalPageCache(fs, pageCache, NULL, layout, Standard.LATEST_RECORD_FORMATS, Configuration.DEFAULT, NullLogService.getInstance(), AdditionalInitialIds.EMPTY, Config.defaults(), INSTANCE)) {
        stores.createNew();
        // when
        int numberOfLabels = 50;
        long nodeId = 0;
        try (NodeImporter importer = new NodeImporter(stores, idMapper, new DataImporter.Monitor(), NULL, INSTANCE)) {
            importer.id(nodeId);
            String[] labels = new String[numberOfLabels];
            for (int i = 0; i < labels.length; i++) {
                labels[i] = "Label" + i;
            }
            importer.labels(labels);
            importer.endOfEntity();
        }
        // then
        NodeStore nodeStore = stores.getNodeStore();
        NodeRecord record = nodeStore.getRecord(nodeId, nodeStore.newRecord(), RecordLoad.NORMAL, CursorContext.NULL);
        long[] labels = NodeLabelsField.parseLabelsField(record).get(nodeStore, CursorContext.NULL);
        assertEquals(numberOfLabels, labels.length);
    }
}

Example 2 with IdMapper

use of org.neo4j.internal.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldHandleLargeAmountsOfDuplicateNodeIds.

@Test
public void shouldHandleLargeAmountsOfDuplicateNodeIds() {
    // GIVEN
    IdMapper mapper = mapper(new LongEncoder(), Radix.LONG, EncodingIdMapper.NO_MONITOR);
    long nodeId = 0;
    int high = 10;
    // a list of input ids
    List<Object> ids = new ArrayList<>();
    for (int run = 0; run < 2; run++) {
        for (long i = 0; i < high / 2; i++) {
            ids.add(high - (i + 1));
            ids.add(i);
        }
    }
    // fed to the IdMapper
    for (Object inputId : ids) {
        mapper.put(inputId, nodeId++, Group.GLOBAL);
    }
    // WHEN
    Collector collector = mock(Collector.class);
    mapper.prepare(values(ids.toArray()), collector, NONE);
    // THEN
    verify(collector, times(high)).collectDuplicateNode(any(Object.class), anyLong(), anyString());
    assertEquals(high, count(mapper.leftOverDuplicateNodesIds()));
}

Example 3 with IdMapper

use of org.neo4j.internal.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups.

@Test
public void shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups() {
    // GIVEN
    final ControlledEncoder encoder = new ControlledEncoder(new LongEncoder());
    final int idsPerGroup = 20;
    int groupCount = 5;
    for (int i = 0; i < groupCount; i++) {
        groups.getOrCreate("Group " + i);
    }
    IdMapper mapper = mapper(encoder, Radix.LONG, EncodingIdMapper.NO_MONITOR, ParallelSort.DEFAULT, numberOfCollisions -> new LongCollisionValues(NumberArrayFactories.HEAP, numberOfCollisions, INSTANCE));
    final AtomicReference<Group> group = new AtomicReference<>();
    PropertyValueLookup ids = (nodeId, cursorContext) -> {
        int groupId = toIntExact(nodeId / idsPerGroup);
        if (groupId == groupCount) {
            return null;
        }
        group.set(groups.get(groupId));
        // i.e. all first 10% in each group collides with all other first 10% in each group
        if (nodeId % idsPerGroup < 2) {
            // Let these colliding values encode into the same eId as well,
            // so that they are definitely marked as collisions
            encoder.useThisIdToEncodeNoMatterWhatComesIn(1234567L);
            return nodeId % idsPerGroup;
        }
        // The other 90% will be accidental collisions for something else
        encoder.useThisIdToEncodeNoMatterWhatComesIn((long) (123456 - group.get().id()));
        return nodeId;
    };
    // WHEN
    int count = idsPerGroup * groupCount;
    for (long nodeId = 0; nodeId < count; nodeId++) {
        mapper.put(ids.lookupProperty(nodeId, NULL), nodeId, group.get());
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // THEN
    verifyNoMoreInteractions(collector);
    for (long nodeId = 0; nodeId < count; nodeId++) {
        assertEquals(nodeId, mapper.get(ids.lookupProperty(nodeId, NULL), group.get()));
    }
    verifyNoMoreInteractions(collector);
    assertFalse(mapper.leftOverDuplicateNodesIds().hasNext());
}

Example 4 with IdMapper

use of org.neo4j.internal.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldEncodeShortStrings.

@Test
public void shouldEncodeShortStrings() {
    // GIVEN
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, EncodingIdMapper.NO_MONITOR);
    // WHEN
    mapper.put("123", 0, Group.GLOBAL);
    mapper.put("456", 1, Group.GLOBAL);
    mapper.prepare(null, mock(Collector.class), NONE);
    // THEN
    assertEquals(1L, mapper.get("456", Group.GLOBAL));
    assertEquals(0L, mapper.get("123", Group.GLOBAL));
}

Example 5 with IdMapper

use of org.neo4j.internal.batchimport.cache.idmapping.IdMapper in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldSkipNullValues.

@Test
public void shouldSkipNullValues() {
    // GIVEN
    MutableLong highDataIndex = new MutableLong();
    MutableLong highTrackerIndex = new MutableLong();
    EncodingIdMapper.Monitor monitor = new EncodingIdMapper.Monitor() {

        @Override
        public void preparing(long highestSetDataIndex, long highestSetTrackerIndex) {
            highDataIndex.setValue(highestSetDataIndex);
            highTrackerIndex.setValue(highestSetTrackerIndex);
        }
    };
    IdMapper idMapper = mapper(new LongEncoder(), Radix.LONG, monitor);
    long count = 1_000;
    for (long id = 0; id < count; id++) {
        long nodeId = id * 2;
        idMapper.put(id, nodeId, Group.GLOBAL);
    }
    // WHEN
    idMapper.prepare((nodeId, cursorContext) -> {
        throw new RuntimeException("Should not be called");
    }, Collector.EMPTY, NONE);
    // THEN
    assertEquals((count - 1) * 2, highDataIndex.longValue());
    assertEquals(count - 1, highTrackerIndex.longValue());
}