Examples with NodeRelationshipCache org.neo4j.internal.batchimport.cache.NodeRelationshipCache used on opensource projects

Example 1 with NodeRelationshipCache

use of org.neo4j.internal.batchimport.cache.NodeRelationshipCache in project neo4j by neo4j.

the class ImportLogicTest method shouldSplitUpRelationshipTypesInBatches.

@Test
void shouldSplitUpRelationshipTypesInBatches() {
    // GIVEN
    int denseNodeThreshold = 5;
    int numberOfNodes = 100;
    int numberOfTypes = 10;
    NodeRelationshipCache cache = new NodeRelationshipCache(NumberArrayFactories.HEAP, denseNodeThreshold, EmptyMemoryTracker.INSTANCE);
    cache.setNodeCount(numberOfNodes + 1);
    Direction[] directions = Direction.values();
    for (int i = 0; i < numberOfNodes; i++) {
        int count = random.nextInt(1, denseNodeThreshold * 2);
        cache.setCount(i, count, random.nextInt(numberOfTypes), random.among(directions));
    }
    cache.countingCompleted();
    List<DataStatistics.RelationshipTypeCount> types = new ArrayList<>();
    int numberOfRelationships = 0;
    for (int i = 0; i < numberOfTypes; i++) {
        int count = random.nextInt(1, 100);
        types.add(new DataStatistics.RelationshipTypeCount(i, count));
        numberOfRelationships += count;
    }
    types.sort((t1, t2) -> Long.compare(t2.getCount(), t1.getCount()));
    DataStatistics typeDistribution = new DataStatistics(0, 0, types.toArray(new DataStatistics.RelationshipTypeCount[0]));
    // WHEN enough memory for all types
    {
        long memory = cache.calculateMaxMemoryUsage(numberOfRelationships) * numberOfTypes;
        int upToType = ImportLogic.nextSetOfTypesThatFitInMemory(typeDistribution, 0, memory, cache.getNumberOfDenseNodes());
        // THEN
        assertEquals(types.size(), upToType);
    }
    // and WHEN less than enough memory for all types
    {
        long memory = cache.calculateMaxMemoryUsage(numberOfRelationships) * numberOfTypes / 3;
        int startingFromType = 0;
        int rounds = 0;
        while (startingFromType < types.size()) {
            rounds++;
            startingFromType = ImportLogic.nextSetOfTypesThatFitInMemory(typeDistribution, startingFromType, memory, cache.getNumberOfDenseNodes());
        }
        assertEquals(types.size(), startingFromType);
        assertThat(rounds).isGreaterThan(1);
    }
}

Example 2 with NodeRelationshipCache

use of org.neo4j.internal.batchimport.cache.NodeRelationshipCache in project neo4j by neo4j.

the class CalculateDenseNodesStepTest method shouldNotProcessLoopsTwice.

@Test
void shouldNotProcessLoopsTwice() throws Exception {
    // GIVEN
    NodeRelationshipCache cache = mock(NodeRelationshipCache.class);
    try (CalculateDenseNodesStep step = new CalculateDenseNodesStep(mock(StageControl.class), Configuration.DEFAULT, cache)) {
        step.processors(4);
        step.start(0);
        // WHEN
        long id = 0;
        RelationshipRecord[] batch = batch(relationship(id++, 1, 5), relationship(id++, 3, 10), // <-- the loop
        relationship(id++, 2, 2), relationship(id, 4, 1));
        step.receive(0, batch);
        step.endOfUpstream();
        step.awaitCompleted();
        // THEN
        verify(cache, times(2)).incrementCount(eq(1L));
        verify(cache).incrementCount(eq(2L));
        verify(cache).incrementCount(eq(3L));
        verify(cache).incrementCount(eq(4L));
        verify(cache).incrementCount(eq(5L));
        verify(cache).incrementCount(eq(10L));
    }
}

Example 3 with NodeRelationshipCache

use of org.neo4j.internal.batchimport.cache.NodeRelationshipCache in project neo4j by neo4j.

the class ImportLogic method initialize.

public void initialize(Input input) throws IOException {
    log.info("Import starting");
    startTime = currentTimeMillis();
    this.input = input;
    PageCacheArrayFactoryMonitor numberArrayFactoryMonitor = new PageCacheArrayFactoryMonitor();
    numberArrayFactory = auto(neoStore.getPageCache(), pageCacheTracer, databaseDirectory, config.allowCacheAllocationOnHeap(), numberArrayFactoryMonitor, log, databaseName);
    // Some temporary caches and indexes in the import
    idMapper = instantiateIdMapper(input);
    nodeRelationshipCache = new NodeRelationshipCache(numberArrayFactory, dbConfig.get(GraphDatabaseSettings.dense_node_threshold), memoryTracker);
    Input.Estimates inputEstimates = input.calculateEstimates(neoStore.getPropertyStore().newValueEncodedSizeCalculator());
    // Sanity checking against estimates
    new EstimationSanityChecker(recordFormats, monitor).sanityCheck(inputEstimates);
    new HeapSizeSanityChecker(monitor).sanityCheck(inputEstimates, recordFormats, neoStore, NodeRelationshipCache.memoryEstimation(inputEstimates.numberOfNodes()), idMapper.memoryEstimation(inputEstimates.numberOfNodes()));
    dependencies.satisfyDependencies(inputEstimates, idMapper, neoStore, nodeRelationshipCache, numberArrayFactoryMonitor);
    if (neoStore.determineDoubleRelationshipRecordUnits(inputEstimates)) {
        monitor.doubleRelationshipRecordUnitsEnabled();
    }
    executionMonitor.initialize(dependencies);
}