Examples with CachedStoreCursors org.neo4j.kernel.impl.store.cursor.CachedStoreCursors used on opensource projects

Example 26 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class RelationshipGroupStoreTest method checkingIfRecordIsInUseMustHappenAfterConsistentRead.

@Test
void checkingIfRecordIsInUseMustHappenAfterConsistentRead() {
    AtomicBoolean nextReadIsInconsistent = new AtomicBoolean(false);
    try (PageCache pageCache = PageCacheSupportExtension.getPageCache(fs, config().withInconsistentReads(nextReadIsInconsistent))) {
        StoreFactory factory = factory(null, pageCache);
        try (NeoStores neoStores = factory.openAllNeoStores(true);
            var storeCursors = new CachedStoreCursors(neoStores, NULL)) {
            RecordStore<RelationshipGroupRecord> relationshipGroupStore = neoStores.getRelationshipGroupStore();
            RelationshipGroupRecord record = new RelationshipGroupRecord(1).initialize(true, 2, 3, 4, 5, 6, Record.NO_NEXT_RELATIONSHIP.intValue());
            try (var storeCursor = storeCursors.writeCursor(GROUP_CURSOR)) {
                relationshipGroupStore.updateRecord(record, storeCursor, CursorContext.NULL, StoreCursors.NULL);
            }
            nextReadIsInconsistent.set(true);
            // Now the following should not throw any RecordNotInUse exceptions
            RelationshipGroupRecord readBack = relationshipGroupStore.getRecordByCursor(1, relationshipGroupStore.newRecord(), NORMAL, storeCursors.readCursor(GROUP_CURSOR));
            assertThat(readBack.toString()).isEqualTo(record.toString());
        }
    }
}

Example 27 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class ConsistencyCheckService method runFullConsistencyCheck.

public Result runFullConsistencyCheck(DatabaseLayout layout, Config config, ProgressMonitorFactory progressFactory, final LogProvider logProvider, final FileSystemAbstraction fileSystem, final PageCache pageCache, DebugContext debugContext, Path reportDir, ConsistencyFlags consistencyFlags, PageCacheTracer pageCacheTracer, MemoryTracker memoryTracker) throws ConsistencyCheckIncompleteException {
    // Right now we only support consistency checker on record storage engine
    RecordDatabaseLayout databaseLayout = RecordDatabaseLayout.convert(layout);
    assertRecovered(databaseLayout, config, fileSystem, memoryTracker);
    Log outLog = logProvider.getLog(getClass());
    config.set(GraphDatabaseSettings.pagecache_warmup_enabled, false);
    LifeSupport life = new LifeSupport();
    final DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory(fileSystem, immediate(), databaseLayout.getDatabaseName());
    DatabaseReadOnlyChecker readOnlyChecker = readOnly();
    StoreFactory factory = new StoreFactory(databaseLayout, config, idGeneratorFactory, pageCache, fileSystem, logProvider, pageCacheTracer, readOnlyChecker);
    ConsistencySummaryStatistics summary;
    final Path reportFile = chooseReportPath(reportDir);
    Log4jLogProvider reportLogProvider = new Log4jLogProvider(LogConfig.createBuilder(fileSystem, reportFile, Level.INFO).createOnDemand().withCategory(false).build());
    Log reportLog = reportLogProvider.getLog(getClass());
    Log log = new DuplicatingLog(outLog, reportLog);
    // Bootstrap kernel extensions
    Monitors monitors = new Monitors();
    JobScheduler jobScheduler = life.add(JobSchedulerFactory.createInitialisedScheduler());
    TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TokenHolder.TYPE_PROPERTY_KEY), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TokenHolder.TYPE_LABEL), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TokenHolder.TYPE_RELATIONSHIP_TYPE));
    final RecoveryCleanupWorkCollector workCollector = RecoveryCleanupWorkCollector.ignore();
    var extensions = life.add(instantiateExtensions(databaseLayout, fileSystem, config, new SimpleLogService(logProvider), pageCache, jobScheduler, workCollector, // We use TOOL context because it's true, and also because it uses the 'single' operational mode, which is important.
    TOOL, monitors, tokenHolders, pageCacheTracer, readOnlyChecker));
    var indexes = life.add(StaticIndexProviderMapFactory.create(life, config, pageCache, fileSystem, new SimpleLogService(logProvider), monitors, readOnlyChecker, TOOL, workCollector, pageCacheTracer, databaseLayout, tokenHolders, jobScheduler, extensions));
    try (NeoStores neoStores = factory.openAllNeoStores()) {
        long lastCommittedTransactionId = neoStores.getMetaDataStore().getLastCommittedTransactionId();
        // Don't start the counts stores here as part of life, instead only shut down. This is because it's better to let FullCheck
        // start it and add its missing/broken detection where it can report to user.
        CountsStoreManager countsStoreManager = life.add(new CountsStoreManager(pageCache, fileSystem, databaseLayout, pageCacheTracer, memoryTracker, logProvider, lastCommittedTransactionId));
        RelationshipGroupDegreesStoreManager groupDegreesStoreManager = life.add(new RelationshipGroupDegreesStoreManager(pageCache, fileSystem, databaseLayout, pageCacheTracer, memoryTracker, logProvider, lastCommittedTransactionId));
        // Load tokens before starting extensions, etc.
        try (var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer(CONSISTENCY_TOKEN_READER_TAG));
            var storeCursors = new CachedStoreCursors(neoStores, cursorContext)) {
            tokenHolders.setInitialTokens(StoreTokens.allReadableTokens(neoStores), storeCursors);
        }
        life.start();
        IndexStatisticsStore indexStatisticsStore = new IndexStatisticsStore(pageCache, databaseLayout, workCollector, readOnlyChecker, pageCacheTracer);
        life.add(indexStatisticsStore);
        int numberOfThreads = defaultConsistencyCheckThreadsNumber();
        DirectStoreAccess stores = new DirectStoreAccess(neoStores, indexes, tokenHolders, indexStatisticsStore, idGeneratorFactory);
        double memoryLimitLeewayFactor = config.get(GraphDatabaseInternalSettings.consistency_check_memory_limit_factor);
        FullCheck check = new FullCheck(progressFactory, numberOfThreads, consistencyFlags, config, debugContext, NodeBasedMemoryLimiter.defaultWithLeeway(memoryLimitLeewayFactor));
        summary = check.execute(pageCache, stores, countsStoreManager, groupDegreesStoreManager, null, pageCacheTracer, memoryTracker, log);
    } finally {
        life.shutdown();
        reportLogProvider.close();
    }
    if (!summary.isConsistent()) {
        log.warn("See '%s' for a detailed consistency report.", reportFile);
        return Result.failure(reportFile, summary);
    }
    return Result.success(reportFile, summary);
}

Example 28 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class IndexChecker method cacheIndex.

private void cacheIndex(IndexContext index, LongRange nodeIdRange, boolean firstRange, CursorContext cursorContext, StoreCursors storeCursors) throws Exception {
    IndexAccessor accessor = indexAccessors.accessorFor(index.descriptor);
    IndexEntriesReader[] partitions = accessor.newAllEntriesValueReader(context.execution.getNumberOfThreads(), cursorContext);
    try {
        Value[][] firstValues = new Value[partitions.length][];
        Value[][] lastValues = new Value[partitions.length][];
        long[] firstEntityIds = new long[partitions.length];
        long[] lastEntityIds = new long[partitions.length];
        ThrowingRunnable[] workers = new ThrowingRunnable[partitions.length];
        for (int i = 0; i < partitions.length; i++) {
            IndexEntriesReader partition = partitions[i];
            int slot = i;
            workers[i] = () -> {
                int lastChecksum = 0;
                int progressPart = 0;
                ProgressListener localCacheProgress = cacheProgress.threadLocalReporter();
                var client = cacheAccess.client();
                try (var context = new CursorContext(this.context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_CACHER_TAG));
                    var localStoreCursors = new CachedStoreCursors(this.context.neoStores, context)) {
                    while (partition.hasNext() && !this.context.isCancelled()) {
                        long entityId = partition.next();
                        if (!nodeIdRange.isWithinRangeExclusiveTo(entityId)) {
                            if (firstRange && entityId >= this.context.highNodeId) {
                                reporter.forIndexEntry(new IndexEntry(index.descriptor, this.context.tokenNameLookup, entityId)).nodeNotInUse(this.context.recordLoader.node(entityId, localStoreCursors));
                            } else if (firstRange && index.descriptor.isUnique() && index.hasValues) {
                                // We check all values belonging to unique indexes while we are checking the first range, to not
                                // miss duplicated values belonging to different ranges.
                                Value[] indexedValues = partition.values();
                                int checksum = checksum(indexedValues);
                                assert checksum <= CHECKSUM_MASK;
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, localStoreCursors, entityId, indexedValues, checksum);
                            }
                            continue;
                        }
                        int data = IN_USE_MASK;
                        if (index.hasValues) {
                            Value[] indexedValues = partition.values();
                            int checksum = checksum(indexedValues);
                            assert checksum <= CHECKSUM_MASK;
                            data |= checksum;
                            // Also take the opportunity to verify uniqueness, if the index is a uniqueness index
                            if (firstRange && index.descriptor.isUnique()) {
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, localStoreCursors, entityId, indexedValues, checksum);
                            }
                        }
                        client.putToCacheSingle(entityId, index.cacheSlotOffset, data);
                        if (++progressPart == INDEX_CACHING_PROGRESS_FACTOR) {
                            localCacheProgress.add(1);
                            progressPart = 0;
                        }
                    }
                }
                localCacheProgress.done();
            };
        }
        // Run the workers that cache the index contents and that do partition-local uniqueness checking, if index is unique
        context.execution.run("Cache index", workers);
        // Then, also if the index is unique then do uniqueness checking of the seams between the partitions
        if (firstRange && index.descriptor.isUnique() && !context.isCancelled()) {
            for (int i = 0; i < partitions.length - 1; i++) {
                Value[] left = lastValues[i];
                Value[] right = firstValues[i + 1];
                // Skip any empty partition - can be empty if all entries in a partition of the index were for nodes outside of the current range.
                if (left != null && right != null && Arrays.equals(left, right)) {
                    long leftEntityId = lastEntityIds[i];
                    long rightEntityId = firstEntityIds[i + 1];
                    reporter.forNode(context.recordLoader.node(leftEntityId, storeCursors)).uniqueIndexNotUnique(index.descriptor, left, rightEntityId);
                }
            }
        }
    } finally {
        IOUtils.closeAll(partitions);
    }
}

Example 29 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class FulltextIndexProviderTest method indexWithUnknownAnalyzerWillBeMarkedAsFailedOnStartup.

@Test
void indexWithUnknownAnalyzerWillBeMarkedAsFailedOnStartup() throws Exception {
    // Create a full-text index.
    long indexId;
    try (KernelTransactionImplementation transaction = getKernelTransaction()) {
        int[] propertyIds = { propIdHa };
        SchemaDescriptor schema = SchemaDescriptors.fulltext(EntityType.NODE, new int[] { labelIdHa }, propertyIds);
        IndexPrototype prototype = IndexPrototype.forSchema(schema).withIndexType(FULLTEXT).withName(NAME);
        SchemaWrite schemaWrite = transaction.schemaWrite();
        IndexDescriptor index = schemaWrite.indexCreate(prototype);
        indexId = index.getId();
        transaction.success();
    }
    // Modify the full-text index such that it has an analyzer configured that does not exist.
    controller.restartDbms(builder -> {
        var cacheTracer = NULL;
        FileSystemAbstraction fs = builder.getFileSystem();
        RecordDatabaseLayout databaseLayout = RecordDatabaseLayout.of(Config.defaults(GraphDatabaseSettings.neo4j_home, builder.getHomeDirectory()));
        DefaultIdGeneratorFactory idGenFactory = new DefaultIdGeneratorFactory(fs, RecoveryCleanupWorkCollector.ignore(), databaseLayout.getDatabaseName());
        try (JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
            PageCache pageCache = StandalonePageCacheFactory.createPageCache(fs, scheduler, cacheTracer)) {
            StoreFactory factory = new StoreFactory(databaseLayout, Config.defaults(), idGenFactory, pageCache, fs, NullLogProvider.getInstance(), cacheTracer, writable());
            var cursorContext = CursorContext.NULL;
            try (NeoStores neoStores = factory.openAllNeoStores(false);
                var storeCursors = new CachedStoreCursors(neoStores, cursorContext)) {
                TokenHolders tokens = StoreTokens.readOnlyTokenHolders(neoStores, storeCursors);
                SchemaStore schemaStore = neoStores.getSchemaStore();
                SchemaStorage storage = new SchemaStorage(schemaStore, tokens, () -> KernelVersion.LATEST);
                IndexDescriptor index = (IndexDescriptor) storage.loadSingleSchemaRule(indexId, storeCursors);
                Map<String, Value> indexConfigMap = new HashMap<>(index.getIndexConfig().asMap());
                for (Map.Entry<String, Value> entry : indexConfigMap.entrySet()) {
                    if (entry.getKey().contains("analyzer")) {
                        // This analyzer does not exist!
                        entry.setValue(Values.stringValue("bla-bla-lyzer"));
                    }
                }
                index = index.withIndexConfig(IndexConfig.with(indexConfigMap));
                storage.writeSchemaRule(index, cursorContext, INSTANCE, storeCursors);
                schemaStore.flush(cursorContext);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return builder;
    });
    // Verify that the index comes up in a failed state.
    try (Transaction tx = db.beginTx()) {
        IndexDefinition index = tx.schema().getIndexByName(NAME);
        Schema.IndexState indexState = tx.schema().getIndexState(index);
        assertThat(indexState).isEqualTo(Schema.IndexState.FAILED);
        String indexFailure = tx.schema().getIndexFailure(index);
        assertThat(indexFailure).contains("bla-bla-lyzer");
    }
    // Verify that the failed index can be dropped.
    try (Transaction tx = db.beginTx()) {
        tx.schema().getIndexByName(NAME).drop();
        assertThrows(IllegalArgumentException.class, () -> tx.schema().getIndexByName(NAME));
        tx.commit();
    }
    try (Transaction tx = db.beginTx()) {
        assertThrows(IllegalArgumentException.class, () -> tx.schema().getIndexByName(NAME));
    }
    controller.restartDbms();
    try (Transaction tx = db.beginTx()) {
        assertThrows(IllegalArgumentException.class, () -> tx.schema().getIndexByName(NAME));
    }
}

Example 30 with CachedStoreCursors

use of org.neo4j.kernel.impl.store.cursor.CachedStoreCursors in project neo4j by neo4j.

the class RelationshipChecker method checkNodesReferencingUnusedRelationships.

private void checkNodesReferencingUnusedRelationships(long fromNodeId, long toNodeId, PageCacheTracer pageCacheTracer) {
    // Do this after we've done node.nextRel caching and checking of those. Checking also clears those values, so simply
    // go through the cache and see if there are any relationship ids left and report them
    CacheAccess.Client client = cacheAccess.client();
    try (var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer(UNUSED_RELATIONSHIP_CHECKER_TAG));
        var storeCursors = new CachedStoreCursors(this.context.neoStores, cursorContext)) {
        for (long id = fromNodeId; id < toNodeId && !context.isCancelled(); id++) {
            // Only check if we haven't come across this sparse node while checking relationships
            boolean nodeInUse = client.getBooleanFromCache(id, CacheSlots.NodeLink.SLOT_IN_USE);
            if (nodeInUse) {
                boolean needsChecking = client.getBooleanFromCache(id, CacheSlots.NodeLink.SLOT_CHECK_MARK);
                if (needsChecking) {
                    long nodeNextRel = client.getFromCache(id, CacheSlots.NodeLink.SLOT_RELATIONSHIP_ID);
                    boolean nodeIsDense = client.getBooleanFromCache(id, CacheSlots.NodeLink.SLOT_IS_DENSE);
                    if (!NULL_REFERENCE.is(nodeNextRel)) {
                        if (!nodeIsDense) {
                            RelationshipRecord relationship = recordLoader.relationship(nodeNextRel, storeCursors);
                            NodeRecord node = recordLoader.node(id, storeCursors);
                            if (!relationship.inUse()) {
                                reporter.forNode(node).relationshipNotInUse(relationship);
                            } else {
                                reporter.forNode(node).relationshipForOtherNode(relationship);
                            }
                        } else {
                            RelationshipGroupRecord group = recordLoader.relationshipGroup(nodeNextRel, storeCursors);
                            if (!group.inUse()) {
                                reporter.forNode(recordLoader.node(id, storeCursors)).relationshipGroupNotInUse(group);
                            } else {
                                reporter.forNode(recordLoader.node(id, storeCursors)).relationshipGroupHasOtherOwner(group);
                            }
                        }
                    }
                }
            }
        }
    }
}