Examples with SimpleStageControl org.neo4j.internal.batchimport.staging.SimpleStageControl used on opensource projects

Example 1 with SimpleStageControl

use of org.neo4j.internal.batchimport.staging.SimpleStageControl in project neo4j by neo4j.

the class ProcessRelationshipCountsDataStepTest method instantiateStep.

private static ProcessRelationshipCountsDataStep instantiateStep(int highLabelId, int highRelationshipTypeId, long labelCacheSize, int maxProcessors, long maxMemory) {
    StageControl control = new SimpleStageControl();
    NodeLabelsCache cache = nodeLabelsCache(labelCacheSize);
    Configuration config = mock(Configuration.class);
    when(config.maxNumberOfProcessors()).thenReturn(maxProcessors);
    when(config.maxMemoryUsage()).thenReturn(maxMemory);
    return new ProcessRelationshipCountsDataStep(control, cache, config, highLabelId, highRelationshipTypeId, mock(CountsAccessor.Updater.class), NumberArrayFactories.OFF_HEAP, ProgressReporter.SILENT, PageCacheTracer.NULL, INSTANCE);
}

Example 2 with SimpleStageControl

use of org.neo4j.internal.batchimport.staging.SimpleStageControl in project neo4j by neo4j.

the class RecordProcessorStepTest method shouldMergeResultsWhenCompleted.

@Test
void shouldMergeResultsWhenCompleted() throws Exception {
    // given
    int numThreads = 4;
    Configuration config = Configuration.DEFAULT;
    MutableLong result = new MutableLong();
    AtomicInteger doneCalls = new AtomicInteger();
    AtomicInteger closeCalls = new AtomicInteger();
    try (RecordProcessorStep<NodeRecord> step = new RecordProcessorStep<>(new SimpleStageControl(), "test", config, () -> new TestProcessor(result, doneCalls, closeCalls), true, numThreads, NULL)) {
        // when
        step.start(0);
        int batchesPerThread = 100;
        AtomicLong nextId = new AtomicLong();
        Race race = new Race().withEndCondition();
        race.addContestants(numThreads, () -> {
            long startId = nextId.getAndAdd(config.batchSize());
            NodeRecord[] batch = new NodeRecord[config.batchSize()];
            for (int r = 0; r < batch.length; r++) {
                batch[r] = new NodeRecord(startId + r);
                batch[r].setInUse(true);
            }
            step.process(batch, null, CursorContext.NULL);
        }, batchesPerThread);
        race.goUnchecked();
        // then
        step.done();
        assertThat(result.longValue()).isEqualTo(numThreads * config.batchSize());
    }
    // then also
    assertThat(doneCalls.longValue()).isEqualTo(1);
    assertThat(closeCalls.longValue()).isEqualTo(numThreads);
}

Example 3 with SimpleStageControl

use of org.neo4j.internal.batchimport.staging.SimpleStageControl in project neo4j by neo4j.

the class DeleteDuplicateNodesStepTest method shouldDeleteEverythingAboutTheDuplicatedNodes.

@RepeatedTest(10)
void shouldDeleteEverythingAboutTheDuplicatedNodes() throws Exception {
    // given
    Ids[] ids = new Ids[9];
    DataImporter.Monitor monitor = new DataImporter.Monitor();
    // node with many properties and many labels
    ids[0] = createNode(monitor, neoStores, 10, 10);
    // node with many properties and few labels
    ids[1] = createNode(monitor, neoStores, 10, 1);
    // node with many properties and no labels
    ids[2] = createNode(monitor, neoStores, 10, 0);
    // node with few properties and many labels
    ids[3] = createNode(monitor, neoStores, 1, 10);
    // node with few properties and few labels
    ids[4] = createNode(monitor, neoStores, 1, 1);
    // node with few properties and no labels
    ids[5] = createNode(monitor, neoStores, 1, 0);
    // node with no properties and many labels
    ids[6] = createNode(monitor, neoStores, 0, 10);
    // node with no properties and few labels
    ids[7] = createNode(monitor, neoStores, 0, 1);
    // node with no properties and no labels
    ids[8] = createNode(monitor, neoStores, 0, 0);
    // when
    long[] duplicateNodeIds = randomNodes(ids);
    SimpleStageControl control = new SimpleStageControl();
    try (DeleteDuplicateNodesStep step = new DeleteDuplicateNodesStep(control, Configuration.DEFAULT, iterator(duplicateNodeIds), neoStores.getNodeStore(), neoStores.getPropertyStore(), monitor, PageCacheTracer.NULL)) {
        control.steps(step);
        startAndAwaitCompletionOf(step);
    }
    control.assertHealthy();
    // then
    int expectedNodes = 0;
    int expectedProperties = 0;
    for (Ids entity : ids) {
        boolean expectedToBeInUse = !ArrayUtils.contains(duplicateNodeIds, entity.node.getId());
        int stride = expectedToBeInUse ? 1 : 0;
        expectedNodes += stride;
        // Verify node record
        assertEquals(expectedToBeInUse, neoStores.getNodeStore().isInUse(entity.node.getId(), NULL));
        // Verify label records
        for (DynamicRecord labelRecord : entity.node.getDynamicLabelRecords()) {
            assertEquals(expectedToBeInUse, neoStores.getNodeStore().getDynamicLabelStore().isInUse(labelRecord.getId(), NULL));
        }
        // Verify property records
        for (PropertyRecord propertyRecord : entity.properties) {
            assertEquals(expectedToBeInUse, neoStores.getPropertyStore().isInUse(propertyRecord.getId(), NULL));
            for (PropertyBlock property : propertyRecord) {
                // Verify property dynamic value records
                for (DynamicRecord valueRecord : property.getValueRecords()) {
                    AbstractDynamicStore valueStore;
                    switch(property.getType()) {
                        case STRING:
                            valueStore = neoStores.getPropertyStore().getStringStore();
                            break;
                        case ARRAY:
                            valueStore = neoStores.getPropertyStore().getArrayStore();
                            break;
                        default:
                            throw new IllegalArgumentException(propertyRecord + " " + property);
                    }
                    assertEquals(expectedToBeInUse, valueStore.isInUse(valueRecord.getId(), NULL));
                }
                expectedProperties += stride;
            }
        }
    }
    assertEquals(expectedNodes, monitor.nodesImported());
    assertEquals(expectedProperties, monitor.propertiesImported());
}

Example 4 with SimpleStageControl

use of org.neo4j.internal.batchimport.staging.SimpleStageControl in project neo4j by neo4j.

the class DeleteDuplicateNodesStepTest method tracePageCacheAccessOnNodeDeduplication.

@Test
void tracePageCacheAccessOnNodeDeduplication() throws Exception {
    // given
    Ids[] ids = new Ids[10];
    DataImporter.Monitor monitor = new DataImporter.Monitor();
    for (int i = 0; i < ids.length; i++) {
        ids[i] = createNode(monitor, neoStores, 1, 1);
    }
    long[] duplicateNodeIds = randomNodes(ids);
    SimpleStageControl control = new SimpleStageControl();
    var cacheTracer = new DefaultPageCacheTracer();
    try (DeleteDuplicateNodesStep step = new DeleteDuplicateNodesStep(control, Configuration.DEFAULT, iterator(duplicateNodeIds), neoStores.getNodeStore(), neoStores.getPropertyStore(), monitor, cacheTracer)) {
        control.steps(step);
        startAndAwaitCompletionOf(step);
    }
    control.assertHealthy();
    // at least 2 events per node is expected since property size is dynamic random thingy
    int expectedEventNumber = duplicateNodeIds.length * 2;
    assertThat(cacheTracer.pins()).isGreaterThanOrEqualTo(expectedEventNumber);
    assertThat(cacheTracer.unpins()).isGreaterThanOrEqualTo(expectedEventNumber);
    assertThat(cacheTracer.hits()).isGreaterThanOrEqualTo(expectedEventNumber);
}

Example 5 with SimpleStageControl

use of org.neo4j.internal.batchimport.staging.SimpleStageControl in project neo4j by neo4j.

the class GenerateIndexUpdatesStepTest method shouldGenerateEntityPropertyUpdatesForRelevantEntityTokens.

@Test
void shouldGenerateEntityPropertyUpdatesForRelevantEntityTokens() throws Exception {
    // given
    StubStorageCursors data = new StubStorageCursors();
    int numNodes = 10;
    MutableLongSet relevantNodeIds = LongSets.mutable.empty();
    for (int i = 0; i < numNodes; i++) {
        int labelId = i % 2 == 0 ? LABEL : OTHER_LABEL;
        data.withNode(i).labels(labelId).properties(KEY, stringValue("name_" + i));
        if (labelId == LABEL) {
            relevantNodeIds.add(i);
        }
    }
    TestPropertyScanConsumer scanConsumer = new TestPropertyScanConsumer();
    GenerateIndexUpdatesStep<StorageNodeCursor> step = new GenerateIndexUpdatesStep<>(new SimpleStageControl(), DEFAULT, data, alwaysTrue(), new NodeCursorBehaviour(data), new int[] { LABEL }, scanConsumer, null, NO_LOCKING, 1, mebiBytes(1), false, PageCacheTracer.NULL, INSTANCE);
    // when
    CapturingBatchSender<GeneratedIndexUpdates> sender = new CapturingBatchSender<>();
    step.process(allNodeIds(data), sender, NULL);
    // then
    GeneratedIndexUpdates updates = sender.batches.get(0);
    updates.completeBatch();
    for (TestPropertyScanConsumer.Record update : scanConsumer.batches.get(0)) {
        assertThat(relevantNodeIds.remove(update.getEntityId())).isTrue();
    }
    assertThat(relevantNodeIds.isEmpty()).isTrue();
}