Examples with CursorContext org.neo4j.io.pagecache.context.CursorContext used on opensource projects

Example 11 with CursorContext

use of org.neo4j.io.pagecache.context.CursorContext in project neo4j by neo4j.

the class RelationshipChecker method check.

private void check(LongRange nodeIdRange, boolean firstRound, long fromRelationshipId, long toRelationshipId, boolean checkToEndOfIndex) throws Exception {
    RelationshipCounter counter = observedCounts.instantiateRelationshipCounter();
    long[] typeHolder = new long[1];
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(RELATIONSHIP_RANGE_CHECKER_TAG));
        RecordReader<RelationshipRecord> relationshipReader = new RecordReader<>(context.neoStores.getRelationshipStore(), true, cursorContext);
        BoundedIterable<EntityTokenRange> relationshipTypeReader = getRelationshipTypeIndexReader(fromRelationshipId, toRelationshipId, checkToEndOfIndex, cursorContext);
        SafePropertyChainReader property = new SafePropertyChainReader(context, cursorContext);
        SchemaComplianceChecker schemaComplianceChecker = new SchemaComplianceChecker(context, mandatoryProperties, indexes, cursorContext, context.memoryTracker)) {
        ProgressListener localProgress = progress.threadLocalReporter();
        CacheAccess.Client client = cacheAccess.client();
        MutableIntObjectMap<Value> propertyValues = new IntObjectHashMap<>();
        Iterator<EntityTokenRange> relationshipTypeRangeIterator = relationshipTypeReader.iterator();
        EntityTokenIndexCheckState typeIndexState = new EntityTokenIndexCheckState(null, fromRelationshipId - 1);
        for (long relationshipId = fromRelationshipId; relationshipId < toRelationshipId && !context.isCancelled(); relationshipId++) {
            localProgress.add(1);
            RelationshipRecord relationshipRecord = relationshipReader.read(relationshipId);
            if (!relationshipRecord.inUse()) {
                continue;
            }
            // Start/end nodes
            long startNode = relationshipRecord.getFirstNode();
            boolean startNodeIsWithinRange = nodeIdRange.isWithinRangeExclusiveTo(startNode);
            boolean startNodeIsNegativeOnFirstRound = startNode < 0 && firstRound;
            if (startNodeIsWithinRange || startNodeIsNegativeOnFirstRound) {
                checkRelationshipVsNode(client, relationshipRecord, startNode, relationshipRecord.isFirstInFirstChain(), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeNotInUse(node), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeDoesNotReferenceBack(node), (relationship, node) -> reporter.forNode(node).relationshipNotFirstInSourceChain(relationship), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeHasNoRelationships(node), relationship -> reporter.forRelationship(relationship).illegalSourceNode(), cursorContext);
            }
            long endNode = relationshipRecord.getSecondNode();
            boolean endNodeIsWithinRange = nodeIdRange.isWithinRangeExclusiveTo(endNode);
            boolean endNodeIsNegativeOnFirstRound = endNode < 0 && firstRound;
            if (endNodeIsWithinRange || endNodeIsNegativeOnFirstRound) {
                checkRelationshipVsNode(client, relationshipRecord, endNode, relationshipRecord.isFirstInSecondChain(), (relationship, node) -> reporter.forRelationship(relationship).targetNodeNotInUse(node), (relationship, node) -> reporter.forRelationship(relationship).targetNodeDoesNotReferenceBack(node), (relationship, node) -> reporter.forNode(node).relationshipNotFirstInTargetChain(relationship), (relationship, node) -> reporter.forRelationship(relationship).targetNodeHasNoRelationships(node), relationship -> reporter.forRelationship(relationship).illegalTargetNode(), cursorContext);
            }
            if (firstRound) {
                if (startNode >= context.highNodeId) {
                    reporter.forRelationship(relationshipRecord).sourceNodeNotInUse(context.recordLoader.node(startNode, cursorContext));
                }
                if (endNode >= context.highNodeId) {
                    reporter.forRelationship(relationshipRecord).targetNodeNotInUse(context.recordLoader.node(endNode, cursorContext));
                }
                // Properties
                typeHolder[0] = relationshipRecord.getType();
                lightClear(propertyValues);
                boolean propertyChainIsOk = property.read(propertyValues, relationshipRecord, reporter::forRelationship, cursorContext);
                if (propertyChainIsOk) {
                    schemaComplianceChecker.checkContainsMandatoryProperties(relationshipRecord, typeHolder, propertyValues, reporter::forRelationship);
                    // gets checked this way, larger indexes will be checked in IndexChecker
                    if (context.consistencyFlags.isCheckIndexes()) {
                        schemaComplianceChecker.checkCorrectlyIndexed((RelationshipRecord) relationshipRecord, typeHolder, propertyValues, reporter::forRelationship);
                    }
                }
                // Type and count
                checkValidToken(relationshipRecord, relationshipRecord.getType(), tokenHolders.relationshipTypeTokens(), neoStores.getRelationshipTypeTokenStore(), (rel, token) -> reporter.forRelationship(rel).illegalRelationshipType(), (rel, token) -> reporter.forRelationship(rel).relationshipTypeNotInUse(token), cursorContext);
                observedCounts.incrementRelationshipTypeCounts(counter, relationshipRecord);
                // Relationship type index
                if (relationshipTypeReader.maxCount() != 0) {
                    checkRelationshipVsRelationshipTypeIndex(relationshipRecord, relationshipTypeRangeIterator, typeIndexState, relationshipId, relationshipRecord.getType(), fromRelationshipId, cursorContext);
                }
            }
            observedCounts.incrementRelationshipNodeCounts(counter, relationshipRecord, startNodeIsWithinRange, endNodeIsWithinRange);
        }
        if (firstRound && !context.isCancelled() && relationshipTypeReader.maxCount() != 0) {
            reportRemainingRelationshipTypeIndexEntries(relationshipTypeRangeIterator, typeIndexState, checkToEndOfIndex ? Long.MAX_VALUE : toRelationshipId, cursorContext);
        }
        localProgress.done();
    }
}

Example 12 with CursorContext

use of org.neo4j.io.pagecache.context.CursorContext in project neo4j by neo4j.

the class SafePropertyChainReader method read.

<PRIMITIVE extends PrimitiveRecord> boolean read(MutableIntObjectMap<Value> intoValues, PRIMITIVE entity, Function<PRIMITIVE, ConsistencyReport.PrimitiveConsistencyReport> primitiveReporter, CursorContext cursorContext) {
    lightClear(seenRecords);
    long propertyRecordId = entity.getNextProp();
    long previousRecordId = NULL_REFERENCE.longValue();
    boolean chainIsOk = true;
    while (!NULL_REFERENCE.is(propertyRecordId) && !context.isCancelled()) {
        if (!seenRecords.add(propertyRecordId)) {
            primitiveReporter.apply(entity).propertyChainContainsCircularReference(propertyReader.record());
            chainIsOk = false;
            break;
        }
        PropertyRecord propertyRecord = propertyReader.read(propertyRecordId);
        if (!propertyRecord.inUse()) {
            primitiveReporter.apply(entity).propertyNotInUse(propertyRecord);
            reporter.forProperty(context.recordLoader.property(previousRecordId, cursorContext)).nextNotInUse(propertyRecord);
            chainIsOk = false;
        } else {
            if (propertyRecord.getPrevProp() != previousRecordId) {
                if (NULL_REFERENCE.is(previousRecordId)) {
                    primitiveReporter.apply(entity).propertyNotFirstInChain(propertyRecord);
                } else {
                    reporter.forProperty(context.recordLoader.property(previousRecordId, cursorContext)).nextDoesNotReferenceBack(propertyRecord);
                // prevDoesNotReferenceBack is not reported, unnecessary double report (same inconsistency from different directions)
                }
                chainIsOk = false;
            }
            for (PropertyBlock block : propertyRecord) {
                int propertyKeyId = block.getKeyIndexId();
                if (!checkValidToken(propertyRecord, propertyKeyId, context.tokenHolders.propertyKeyTokens(), neoStores.getPropertyKeyTokenStore(), (property, token) -> reporter.forProperty(property).invalidPropertyKey(block), (property, token) -> reporter.forProperty(property).keyNotInUse(block, token), cursorContext)) {
                    chainIsOk = false;
                }
                PropertyType type = block.forceGetType();
                Value value = Values.NO_VALUE;
                if (type == null) {
                    reporter.forProperty(propertyRecord).invalidPropertyType(block);
                } else {
                    try {
                        switch(type) {
                            case STRING:
                                dynamicRecords.clear();
                                if (safeLoadDynamicRecordChain(record -> dynamicRecords.add(record.copy()), stringReader, seenDynamicRecordIds, block.getSingleValueLong(), stringStoreBlockSize, NO_DYNAMIC_HANDLER, (id, record) -> reporter.forProperty(propertyRecord).stringNotInUse(block, record), (id, record) -> reporter.forDynamicBlock(RecordType.STRING_PROPERTY, stringReader.record()).nextNotInUse(record), (id, record) -> reporter.forProperty(propertyRecord).stringEmpty(block, record), record -> reporter.forDynamicBlock(RecordType.STRING_PROPERTY, record).recordNotFullReferencesNext(), record -> reporter.forDynamicBlock(RecordType.STRING_PROPERTY, record).invalidLength())) {
                                    value = propertyStore.getTextValueFor(dynamicRecords, cursorContext);
                                }
                                break;
                            case ARRAY:
                                dynamicRecords.clear();
                                if (safeLoadDynamicRecordChain(record -> dynamicRecords.add(record.copy()), arrayReader, seenDynamicRecordIds, block.getSingleValueLong(), arrayStoreBlockSize, NO_DYNAMIC_HANDLER, (id, record) -> reporter.forProperty(propertyRecord).arrayNotInUse(block, record), (id, record) -> reporter.forDynamicBlock(RecordType.ARRAY_PROPERTY, arrayReader.record()).nextNotInUse(record), (id, record) -> reporter.forProperty(propertyRecord).arrayEmpty(block, record), record -> reporter.forDynamicBlock(RecordType.ARRAY_PROPERTY, record).recordNotFullReferencesNext(), record -> reporter.forDynamicBlock(RecordType.ARRAY_PROPERTY, record).invalidLength())) {
                                    value = propertyStore.getArrayFor(dynamicRecords, cursorContext);
                                }
                                break;
                            default:
                                value = type.value(block, null, cursorContext);
                                break;
                        }
                    } catch (Exception e) {
                        reporter.forProperty(propertyRecord).invalidPropertyValue(propertyRecord.getId(), block.getKeyIndexId());
                    }
                }
                if (value == Values.NO_VALUE) {
                    chainIsOk = false;
                } else if (propertyKeyId >= 0 && intoValues.put(propertyKeyId, value) != null) {
                    primitiveReporter.apply(entity).propertyKeyNotUniqueInChain();
                    chainIsOk = false;
                }
            }
        }
        previousRecordId = propertyRecordId;
        propertyRecordId = propertyRecord.getNextProp();
    }
    return chainIsOk;
}

Example 13 with CursorContext

use of org.neo4j.io.pagecache.context.CursorContext in project neo4j by neo4j.

the class IndexChecker method checkVsEntities.

private void checkVsEntities(List<IndexContext> indexes, long fromEntityId, long toEntityId) {
    // This is one thread
    CheckerContext noReportingContext = context.withoutReporting();
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_ENTITY_CHECK_TAG));
        RecordReader<NodeRecord> nodeReader = new RecordReader<>(context.neoStores.getNodeStore(), true, cursorContext);
        RecordReader<DynamicRecord> labelReader = new RecordReader<>(context.neoStores.getNodeStore().getDynamicLabelStore(), false, cursorContext);
        SafePropertyChainReader propertyReader = new SafePropertyChainReader(noReportingContext, cursorContext)) {
        ProgressListener localScanProgress = scanProgress.threadLocalReporter();
        IntObjectHashMap<Value> allValues = new IntObjectHashMap<>();
        var client = cacheAccess.client();
        int numberOfIndexes = indexes.size();
        for (long entityId = fromEntityId; entityId < toEntityId && !context.isCancelled(); entityId++) {
            NodeRecord nodeRecord = nodeReader.read(entityId);
            if (nodeRecord.inUse()) {
                long[] entityTokens = safeGetNodeLabels(noReportingContext, nodeRecord.getId(), nodeRecord.getLabelField(), labelReader, cursorContext);
                lightClear(allValues);
                boolean propertyChainRead = entityTokens != null && propertyReader.read(allValues, nodeRecord, noReportingContext.reporter::forNode, cursorContext);
                if (propertyChainRead) {
                    for (int i = 0; i < numberOfIndexes; i++) {
                        IndexContext index = indexes.get(i);
                        IndexDescriptor descriptor = index.descriptor;
                        long cachedValue = client.getFromCache(entityId, i);
                        boolean nodeIsInIndex = (cachedValue & IN_USE_MASK) != 0;
                        Value[] values = RecordLoading.entityIntersectionWithSchema(entityTokens, allValues, descriptor.schema());
                        if (index.descriptor.schema().isFulltextSchemaDescriptor()) {
                            // The strategy for fulltext indexes is way simpler. Simply check of the sets of tokens (label tokens and property key tokens)
                            // and if they match the index schema descriptor then the node should be in the index, otherwise not
                            int[] nodePropertyKeys = allValues.keySet().toArray();
                            int[] indexPropertyKeys = index.descriptor.schema().getPropertyIds();
                            boolean nodeShouldBeInIndex = index.descriptor.schema().isAffected(entityTokens) && containsAny(indexPropertyKeys, nodePropertyKeys) && valuesContainTextProperty(values);
                            if (nodeShouldBeInIndex && !nodeIsInIndex) {
                                getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, new Object[0]);
                            } else if (!nodeShouldBeInIndex && nodeIsInIndex) {
                                // Fulltext indexes created before 4.3.0-drop02 can contain empty documents (added when the schema matched but the values
                                // were not text). The index still works with those empty documents present, so we don't want to report them as
                                // inconsistencies and force rebuilds.
                                // This utilizes the fact that countIndexedEntities in FulltextIndexReader with non-text values will ask
                                // about documents that doesn't contain those property keys - a document found by that query should be an empty
                                // document we just want to ignore.
                                Value[] noValues = new Value[indexPropertyKeys.length];
                                Arrays.fill(noValues, NO_VALUE);
                                long docsWithNoneOfProperties = indexAccessors.readers().reader(descriptor).countIndexedEntities(entityId, cursorContext, indexPropertyKeys, noValues);
                                if (docsWithNoneOfProperties != 1) {
                                    getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, new Object[0]);
                                }
                            }
                        } else {
                            if (values != null) {
                                // This node should really be in the index, is it?
                                if (!nodeIsInIndex) {
                                    // It wasn't, report it
                                    getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, Values.asObjects(values));
                                } else if (index.hasValues) {
                                    int cachedChecksum = (int) cachedValue & CHECKSUM_MASK;
                                    int actualChecksum = checksum(values);
                                    if (cachedChecksum != actualChecksum) {
                                        getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, Values.asObjects(values));
                                    }
                                }
                            } else {
                                if (nodeIsInIndex) {
                                    reporter.forIndexEntry(new IndexEntry(descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(context.recordLoader.node(entityId, cursorContext));
                                }
                            }
                        }
                    }
                }
            // else this would be reported elsewhere
            } else {
                // This node shouldn't be in any index
                for (int i = 0; i < numberOfIndexes; i++) {
                    boolean isInIndex = (client.getFromCache(entityId, i) & IN_USE_MASK) != 0;
                    if (isInIndex) {
                        reporter.forIndexEntry(new IndexEntry(indexes.get(i).descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(context.recordLoader.node(entityId, cursorContext));
                    }
                }
            }
            localScanProgress.add(1);
        }
        localScanProgress.done();
    }
}

Example 14 with CursorContext

use of org.neo4j.io.pagecache.context.CursorContext in project neo4j by neo4j.

the class FullScanNonUniqueIndexSamplerTest method tracePageCacheAccessOnSampling.

@Test
void tracePageCacheAccessOnSampling() throws IOException {
    Value[] values = generateNumberValues();
    buildTree(values);
    var pageCacheTracer = new DefaultPageCacheTracer();
    var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer("testTracer"));
    assertZeroCursor(cursorContext);
    try (GBPTree<GenericKey, NativeIndexValue> gbpTree = getTree()) {
        FullScanNonUniqueIndexSampler<GenericKey, NativeIndexValue> sampler = new FullScanNonUniqueIndexSampler<>(gbpTree, layout);
        sampler.sample(cursorContext);
    }
    PageCursorTracer cursorTracer = cursorContext.getCursorTracer();
    assertThat(cursorTracer.pins()).isEqualTo(1);
    assertThat(cursorTracer.unpins()).isEqualTo(1);
    assertThat(cursorTracer.faults()).isEqualTo(1);
}

Example 15 with CursorContext

use of org.neo4j.io.pagecache.context.CursorContext in project neo4j by neo4j.

the class IndexStatisticsStoreTest method tracePageCacheAccessOnCheckpoint.

@Test
void tracePageCacheAccessOnCheckpoint() throws IOException {
    var cacheTracer = new DefaultPageCacheTracer();
    var store = openStore(cacheTracer, "checkpoint");
    try (var cursorContext = new CursorContext(cacheTracer.createPageCursorTracer("tracePageCacheAccessOnCheckpoint"))) {
        for (int i = 0; i < 100; i++) {
            store.replaceStats(i, new IndexSample());
        }
        store.checkpoint(cursorContext);
        PageCursorTracer cursorTracer = cursorContext.getCursorTracer();
        assertThat(cursorTracer.pins()).isEqualTo(43);
        assertThat(cursorTracer.unpins()).isEqualTo(43);
        assertThat(cursorTracer.hits()).isEqualTo(35);
        assertThat(cursorTracer.faults()).isEqualTo(8);
    }
}