Examples with CachedStoreCursors org.neo4j.kernel.impl.store.cursor.CachedStoreCursors used on opensource projects

Example 16 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class TestGrowingFileMemoryMapping method shouldGrowAFileWhileContinuingToMemoryMapNewRegions.

@Test
@DisabledOnOs(OS.WINDOWS)
void shouldGrowAFileWhileContinuingToMemoryMapNewRegions() {
    // given
    final int NUMBER_OF_RECORDS = 1000000;
    Config config = Config.defaults();
    DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory(testDirectory.getFileSystem(), immediate(), databaseLayout.getDatabaseName());
    StoreFactory storeFactory = new StoreFactory(databaseLayout, config, idGeneratorFactory, pageCache, testDirectory.getFileSystem(), NullLogProvider.getInstance(), NULL, writable());
    try (NeoStores neoStores = storeFactory.openAllNeoStores(true);
        var storeCursors = new CachedStoreCursors(neoStores, CursorContext.NULL)) {
        NodeStore nodeStore = neoStores.getNodeStore();
        // when
        int iterations = 2 * NUMBER_OF_RECORDS;
        long startingId = nodeStore.nextId(CursorContext.NULL);
        long nodeId = startingId;
        try (PageCursor pageCursor = storeCursors.writeCursor(NODE_CURSOR)) {
            for (int i = 0; i < iterations; i++) {
                NodeRecord record = new NodeRecord(nodeId).initialize(false, 0, false, i, 0);
                record.setInUse(true);
                nodeStore.updateRecord(record, pageCursor, CursorContext.NULL, StoreCursors.NULL);
                nodeId = nodeStore.nextId(CursorContext.NULL);
            }
        }
        // then
        NodeRecord record = new NodeRecord(0).initialize(false, 0, false, 0, 0);
        var pageCursor = storeCursors.readCursor(NODE_CURSOR);
        for (int i = 0; i < iterations; i++) {
            record.setId(startingId + i);
            nodeStore.getRecordByCursor(i, record, NORMAL, pageCursor);
            assertTrue(record.inUse(), "record[" + i + "] should be in use");
            assertThat(record.getNextRel()).as("record[" + i + "] should have nextRelId of " + i).isEqualTo(i);
        }
    }
}

Example 17 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class FulltextIndexConsistencyCheckIT method mustDiscoverRelationshipInIndexMissingFromStore.

@Disabled("Turns out that this is not something that the consistency checker actually looks for, currently. " + "The test is disabled until the consistency checker is extended with checks that will discover this sort of inconsistency.")
@Test
void mustDiscoverRelationshipInIndexMissingFromStore() throws Exception {
    GraphDatabaseService db = createDatabase();
    try (Transaction tx = db.beginTx()) {
        tx.execute(format(RELATIONSHIP_CREATE, "rels", asStrList("REL"), asStrList("prop"))).close();
        tx.commit();
    }
    long relId;
    try (Transaction tx = db.beginTx()) {
        tx.schema().awaitIndexesOnline(2, TimeUnit.MINUTES);
        Node node = tx.createNode();
        Relationship rel = node.createRelationshipTo(node, RelationshipType.withName("REL"));
        relId = rel.getId();
        rel.setProperty("prop", "value");
        tx.commit();
    }
    NeoStores stores = getNeoStores(db);
    try (CachedStoreCursors storeCursors = new CachedStoreCursors(stores, NULL)) {
        RelationshipStore relationshipStore = stores.getRelationshipStore();
        RelationshipRecord record = relationshipStore.newRecord();
        PropertyStore propertyStore = stores.getPropertyStore();
        relationshipStore.getRecordByCursor(relId, record, RecordLoad.NORMAL, storeCursors.readCursor(RELATIONSHIP_CURSOR));
        long propId = record.getNextProp();
        record.setNextProp(AbstractBaseRecord.NO_ID);
        try (var storeCursor = storeCursors.writeCursor(RELATIONSHIP_CURSOR)) {
            relationshipStore.updateRecord(record, storeCursor, NULL, storeCursors);
        }
        PropertyRecord propRecord = propertyStore.newRecord();
        propertyStore.getRecordByCursor(propId, propertyStore.newRecord(), RecordLoad.NORMAL, storeCursors.readCursor(PROPERTY_CURSOR));
        propRecord.setInUse(false);
        try (var cursor = storeCursors.writeCursor(PROPERTY_CURSOR)) {
            propertyStore.updateRecord(propRecord, cursor, NULL, storeCursors);
        }
    }
    managementService.shutdown();
    ConsistencyCheckService.Result result = checkConsistency();
    assertFalse(result.isSuccessful());
}

Example 18 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class RecordStorageConsistencyChecker method check.

public void check() throws ConsistencyCheckIncompleteException {
    assert !context.isCancelled();
    try {
        context.initialize();
        // Starting by loading all tokens from store into the TokenHolders, loaded in a safe way of course
        safeLoadTokens(neoStores);
        // Check schema - constraints and indexes, that sort of thing
        // This is done before instantiating the other checker instances because the schema checker will also
        // populate maps regarding mandatory properties which the node/relationship checkers uses
        SchemaChecker schemaChecker = new SchemaChecker(context);
        MutableIntObjectMap<MutableIntSet> mandatoryNodeProperties = new IntObjectHashMap<>();
        MutableIntObjectMap<MutableIntSet> mandatoryRelationshipProperties = new IntObjectHashMap<>();
        try (var cursorContext = new CursorContext(cacheTracer.createPageCursorTracer(SCHEMA_CONSISTENCY_CHECKER_TAG));
            var storeCursors = new CachedStoreCursors(context.neoStores, cursorContext)) {
            schemaChecker.check(mandatoryNodeProperties, mandatoryRelationshipProperties, cursorContext, storeCursors);
        }
        // Some pieces of check logic are extracted from this main class to reduce the size of this class. Instantiate those here first
        NodeChecker nodeChecker = new NodeChecker(context, mandatoryNodeProperties);
        IndexChecker indexChecker = new IndexChecker(context, EntityType.NODE);
        RelationshipChecker relationshipChecker = new RelationshipChecker(context, mandatoryRelationshipProperties);
        RelationshipGroupChecker relationshipGroupChecker = new RelationshipGroupChecker(context);
        RelationshipChainChecker relationshipChainChecker = new RelationshipChainChecker(context);
        ProgressMonitorFactory.Completer progressCompleter = progress.build();
        int numberOfRanges = limiter.numberOfRanges();
        for (int i = 1; limiter.hasNext(); i++) {
            if (isCancelled()) {
                break;
            }
            LongRange range = limiter.next();
            if (numberOfRanges > 1) {
                context.debug("=== Checking range %d/%d (%s) ===", i, numberOfRanges, range);
            }
            context.initializeRange();
            // Tell the cache that the pivot node id is the low end of this range. This will make all interactions with the cache
            // take that into consideration when working with offset arrays where the index is based on node ids.
            cacheAccess.setPivotId(range.from());
            // Go into a node-centric mode where the nodes themselves are checked and somewhat cached off-heap.
            // Then while we have the nodes loaded in cache do all other checking that has anything to do with nodes
            // so that the "other" store can be checked sequentially and the random node lookups will be cheap
            context.runIfAllowed(indexChecker, range);
            cacheAccess.setCacheSlotSizesAndClear(DEFAULT_SLOT_SIZES);
            context.runIfAllowed(nodeChecker, range);
            context.runIfAllowed(relationshipGroupChecker, range);
            context.runIfAllowed(relationshipChecker, range);
            context.runIfAllowed(relationshipChainChecker, range);
        }
        if (!isCancelled() && context.consistencyFlags.isCheckGraph()) {
            // All counts we've observed while doing other checking along the way we compare against the counts store here
            checkCounts();
        }
        progressCompleter.close();
    } catch (Exception e) {
        cancel("ConsistencyChecker failed unexpectedly");
        throw new ConsistencyCheckIncompleteException(e);
    }
}

Example 19 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class RelationshipChainChecker method relationshipVsRelationshipChecker.

private ThrowingRunnable relationshipVsRelationshipChecker(LongRange nodeIdRange, ScanDirection direction, RelationshipStore store, ArrayBlockingQueue<BatchedRelationshipRecords> queue, AtomicBoolean end, int threadId) {
    final RelationshipRecord relationship = store.newRecord();
    final RelationshipRecord otherRelationship = store.newRecord();
    final CacheAccess.Client client = cacheAccess.client();
    final RelationshipLink sourceCachePointer = direction.sourceLink;
    final RelationshipLink targetCachePointer = direction.targetLink;
    final long prevOrNext = direction.cacheSlot;
    return () -> {
        try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(RELATIONSHIP_CONSISTENCY_CHECKER_TAG));
            var storeCursors = new CachedStoreCursors(context.neoStores, cursorContext)) {
            while ((!end.get() || !queue.isEmpty()) && !context.isCancelled()) {
                BatchedRelationshipRecords batch = queue.poll(100, TimeUnit.MILLISECONDS);
                if (batch != null) {
                    while (batch.fillNext(relationship) && !context.isCancelled()) {
                        long firstNode = relationship.getFirstNode();
                        long secondNode = relationship.getSecondNode();
                        // Intentionally not checking nodes outside highId of node store because RelationshipChecker will spot this inconsistency
                        boolean processStartNode = Math.abs(firstNode % numberOfChainCheckers) == threadId && nodeIdRange.isWithinRangeExclusiveTo(firstNode);
                        boolean processEndNode = Math.abs(secondNode % numberOfChainCheckers) == threadId && nodeIdRange.isWithinRangeExclusiveTo(secondNode);
                        if (processStartNode) {
                            checkRelationshipLink(direction, SOURCE_PREV, relationship, client, otherRelationship, store, storeCursors);
                            checkRelationshipLink(direction, SOURCE_NEXT, relationship, client, otherRelationship, store, storeCursors);
                        }
                        if (processEndNode) {
                            checkRelationshipLink(direction, TARGET_PREV, relationship, client, otherRelationship, store, storeCursors);
                            checkRelationshipLink(direction, TARGET_NEXT, relationship, client, otherRelationship, store, storeCursors);
                        }
                        if (processStartNode) {
                            boolean wasInUse = client.getBooleanFromCache(firstNode, SLOT_IN_USE);
                            long link = sourceCachePointer.link(relationship);
                            if (link < NULL_REFERENCE.longValue()) {
                                sourceCachePointer.reportDoesNotReferenceBack(reporter, relationship, otherRelationship);
                            } else {
                                client.putToCache(firstNode, relationship.getId(), link, SOURCE, prevOrNext, 1, longOf(wasInUse), longOf(relationship.isFirstInFirstChain()));
                            }
                        }
                        if (processEndNode) {
                            boolean wasInUse = client.getBooleanFromCache(secondNode, SLOT_IN_USE);
                            long link = targetCachePointer.link(relationship);
                            if (link < NULL_REFERENCE.longValue()) {
                                targetCachePointer.reportDoesNotReferenceBack(reporter, relationship, otherRelationship);
                            } else {
                                client.putToCache(secondNode, relationship.getId(), link, TARGET, prevOrNext, 1, longOf(wasInUse), longOf(relationship.isFirstInSecondChain()));
                            }
                        }
                    }
                }
            }
        }
    };
}

Example 20 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class RelationshipChecker method check.

private void check(LongRange nodeIdRange, boolean firstRound, long fromRelationshipId, long toRelationshipId, boolean checkToEndOfIndex) throws Exception {
    RelationshipCounter counter = observedCounts.instantiateRelationshipCounter();
    long[] typeHolder = new long[1];
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(RELATIONSHIP_RANGE_CHECKER_TAG));
        var storeCursors = new CachedStoreCursors(this.context.neoStores, cursorContext);
        RecordReader<RelationshipRecord> relationshipReader = new RecordReader<>(context.neoStores.getRelationshipStore(), true, cursorContext);
        BoundedIterable<EntityTokenRange> relationshipTypeReader = getRelationshipTypeIndexReader(fromRelationshipId, toRelationshipId, checkToEndOfIndex, cursorContext);
        SafePropertyChainReader property = new SafePropertyChainReader(context, cursorContext);
        SchemaComplianceChecker schemaComplianceChecker = new SchemaComplianceChecker(context, mandatoryProperties, indexes, cursorContext, storeCursors, context.memoryTracker)) {
        ProgressListener localProgress = progress.threadLocalReporter();
        CacheAccess.Client client = cacheAccess.client();
        MutableIntObjectMap<Value> propertyValues = new IntObjectHashMap<>();
        Iterator<EntityTokenRange> relationshipTypeRangeIterator = relationshipTypeReader.iterator();
        EntityTokenIndexCheckState typeIndexState = new EntityTokenIndexCheckState(null, fromRelationshipId - 1);
        for (long relationshipId = fromRelationshipId; relationshipId < toRelationshipId && !context.isCancelled(); relationshipId++) {
            localProgress.add(1);
            RelationshipRecord relationshipRecord = relationshipReader.read(relationshipId);
            if (!relationshipRecord.inUse()) {
                continue;
            }
            // Start/end nodes
            long startNode = relationshipRecord.getFirstNode();
            boolean startNodeIsWithinRange = nodeIdRange.isWithinRangeExclusiveTo(startNode);
            boolean startNodeIsNegativeOnFirstRound = startNode < 0 && firstRound;
            if (startNodeIsWithinRange || startNodeIsNegativeOnFirstRound) {
                checkRelationshipVsNode(client, relationshipRecord, startNode, relationshipRecord.isFirstInFirstChain(), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeNotInUse(node), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeDoesNotReferenceBack(node), (relationship, node) -> reporter.forNode(node).relationshipNotFirstInSourceChain(relationship), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeHasNoRelationships(node), relationship -> reporter.forRelationship(relationship).illegalSourceNode(), storeCursors);
            }
            long endNode = relationshipRecord.getSecondNode();
            boolean endNodeIsWithinRange = nodeIdRange.isWithinRangeExclusiveTo(endNode);
            boolean endNodeIsNegativeOnFirstRound = endNode < 0 && firstRound;
            if (endNodeIsWithinRange || endNodeIsNegativeOnFirstRound) {
                checkRelationshipVsNode(client, relationshipRecord, endNode, relationshipRecord.isFirstInSecondChain(), (relationship, node) -> reporter.forRelationship(relationship).targetNodeNotInUse(node), (relationship, node) -> reporter.forRelationship(relationship).targetNodeDoesNotReferenceBack(node), (relationship, node) -> reporter.forNode(node).relationshipNotFirstInTargetChain(relationship), (relationship, node) -> reporter.forRelationship(relationship).targetNodeHasNoRelationships(node), relationship -> reporter.forRelationship(relationship).illegalTargetNode(), storeCursors);
            }
            if (firstRound) {
                if (startNode >= context.highNodeId) {
                    reporter.forRelationship(relationshipRecord).sourceNodeNotInUse(context.recordLoader.node(startNode, storeCursors));
                }
                if (endNode >= context.highNodeId) {
                    reporter.forRelationship(relationshipRecord).targetNodeNotInUse(context.recordLoader.node(endNode, storeCursors));
                }
                // Properties
                typeHolder[0] = relationshipRecord.getType();
                lightClear(propertyValues);
                boolean propertyChainIsOk = property.read(propertyValues, relationshipRecord, reporter::forRelationship, storeCursors);
                if (propertyChainIsOk) {
                    schemaComplianceChecker.checkContainsMandatoryProperties(relationshipRecord, typeHolder, propertyValues, reporter::forRelationship);
                    // gets checked this way, larger indexes will be checked in IndexChecker
                    if (context.consistencyFlags.isCheckIndexes()) {
                        schemaComplianceChecker.checkCorrectlyIndexed(relationshipRecord, typeHolder, propertyValues, reporter::forRelationship);
                    }
                }
                // Type and count
                checkValidToken(relationshipRecord, relationshipRecord.getType(), tokenHolders.relationshipTypeTokens(), neoStores.getRelationshipTypeTokenStore(), (rel, token) -> reporter.forRelationship(rel).illegalRelationshipType(), (rel, token) -> reporter.forRelationship(rel).relationshipTypeNotInUse(token), storeCursors);
                observedCounts.incrementRelationshipTypeCounts(counter, relationshipRecord);
                // Relationship type index
                if (relationshipTypeReader.maxCount() != 0) {
                    checkRelationshipVsRelationshipTypeIndex(relationshipRecord, relationshipTypeRangeIterator, typeIndexState, relationshipId, relationshipRecord.getType(), fromRelationshipId, storeCursors);
                }
            }
            observedCounts.incrementRelationshipNodeCounts(counter, relationshipRecord, startNodeIsWithinRange, endNodeIsWithinRange);
        }
        if (firstRound && !context.isCancelled() && relationshipTypeReader.maxCount() != 0) {
            reportRemainingRelationshipTypeIndexEntries(relationshipTypeRangeIterator, typeIndexState, checkToEndOfIndex ? Long.MAX_VALUE : toRelationshipId, storeCursors);
        }
        localProgress.done();
    }
}