Examples with RecordDatabaseLayout org.neo4j.io.layout.recordstorage.RecordDatabaseLayout used on opensource projects

Example 26 with RecordDatabaseLayout

use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.

the class ConsistencyCheckWithCorruptGBPTreeIT method multipleCorruptionsInFusionIndex.

@Test
void multipleCorruptionsInFusionIndex() throws Exception {
    // Because NATIVE30 provider use Lucene internally we can not use the snapshot from ephemeral file system because
    // lucene will not use it to store the files. Therefor we use a default file system together with TestDirectory
    // for cleanup.
    final DefaultFileSystemAbstraction fs = new DefaultFileSystemAbstraction();
    final TestDirectory testDirectory = TestDirectory.testDirectory(fs);
    testDirectory.prepareDirectory(ConsistencyCheckWithCorruptGBPTreeIT.class, "multipleCorruptionsInFusionIndex");
    try {
        final Path neo4jHome = testDirectory.homePath();
        dbmsAction(neo4jHome, fs, NATIVE30, db -> {
            Label label = Label.label("label2");
            indexWithNumberData(db, label);
        }, builder -> {
        });
        RecordDatabaseLayout layout = RecordDatabaseLayout.of(Config.defaults(neo4j_home, neo4jHome));
        final Path[] indexFiles = schemaIndexFiles(fs, layout.databaseDirectory(), NATIVE30);
        final List<Path> files = corruptIndexes(fs, readOnly(), (tree, inspection) -> {
            long leafNode = inspection.getLeafNodes().get(1);
            long internalNode = inspection.getInternalNodes().get(0);
            tree.unsafe(pageSpecificCorruption(leafNode, GBPTreeCorruption.rightSiblingPointToNonExisting()), CursorContext.NULL);
            tree.unsafe(pageSpecificCorruption(internalNode, GBPTreeCorruption.setChild(0, internalNode)), CursorContext.NULL);
        }, indexFiles);
        assertTrue(files.size() > 0, "Expected number of corrupted files to be more than one.");
        ConsistencyCheckService.Result result = runConsistencyCheck(fs, neo4jHome, layout, NullLogProvider.getInstance(), NONE, DEFAULT);
        for (Path file : files) {
            assertResultContainsMessage(fs, result, "Index will be excluded from further consistency checks. Index file: " + file.toAbsolutePath());
        }
    } finally {
        testDirectory.cleanup();
    }
}

Example 27 with RecordDatabaseLayout

use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.

the class RecordStorageEngineFactory method loadSchemaRules.

@Override
public List<SchemaRule> loadSchemaRules(FileSystemAbstraction fs, PageCache pageCache, Config config, DatabaseLayout layout, CursorContext cursorContext) {
    RecordDatabaseLayout databaseLayout = convert(layout);
    StoreFactory factory = new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, NullLogProvider.getInstance(), PageCacheTracer.NULL, readOnly());
    try (NeoStores stores = factory.openNeoStores(false, StoreType.SCHEMA, StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY);
        var storeCursors = new CachedStoreCursors(stores, cursorContext)) {
        stores.start(cursorContext);
        TokenHolders tokenHolders = tokenHoldersForSchemaStore(stores, new ReadOnlyTokenCreator(), storeCursors);
        List<SchemaRule> rules = new ArrayList<>();
        new SchemaStorage(stores.getSchemaStore(), tokenHolders, () -> KernelVersion.LATEST).getAll(storeCursors).forEach(rules::add);
        return rules;
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 28 with RecordDatabaseLayout

use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.

the class DegreesRebuildFromStoreTest method skipNotUsedRecordsOnDegreeStoreRebuild.

@Test
void skipNotUsedRecordsOnDegreeStoreRebuild() throws Exception {
    // given a dataset containing mixed sparse and dense nodes with relationships in random directions,
    // where some chains have been marked as having external degrees
    int denseThreshold = dense_node_threshold.defaultValue();
    RecordDatabaseLayout layout = RecordDatabaseLayout.ofFlat(directory.homePath());
    int[] relationshipTypes;
    MutableLongLongMap expectedDegrees = LongLongMaps.mutable.empty();
    Config config = config(denseThreshold);
    try (Lifespan life = new Lifespan()) {
        RecordStorageEngine storageEngine = openStorageEngine(layout, config);
        relationshipTypes = createRelationshipTypes(storageEngine);
        life.add(storageEngine);
        generateData(storageEngine, denseThreshold, relationshipTypes);
        storageEngine.relationshipGroupDegreesStore().accept((groupId, direction, degree) -> expectedDegrees.put(combinedKeyOnGroupAndDirection(groupId, direction), degree), NULL);
        assertThat(expectedDegrees.isEmpty()).isFalse();
        RelationshipGroupStore groupStore = storageEngine.testAccessNeoStores().getRelationshipGroupStore();
        try (StoreCursors storageCursors = storageEngine.createStorageCursors(NULL)) {
            long highId = groupStore.getHighId();
            assertThat(highId).isGreaterThan(1);
            for (int i = 10; i < highId; i++) {
                RelationshipGroupRecord record = groupStore.getRecordByCursor(i, new RelationshipGroupRecord(i), RecordLoad.ALWAYS, storageCursors.readCursor(GROUP_CURSOR));
                expectedDegrees.remove(combinedKeyOnGroupAndDirection(record.getId(), RelationshipDirection.OUTGOING));
                expectedDegrees.remove(combinedKeyOnGroupAndDirection(record.getId(), RelationshipDirection.INCOMING));
                expectedDegrees.remove(combinedKeyOnGroupAndDirection(record.getId(), RelationshipDirection.LOOP));
                record.setInUse(false);
                try (var groupStoreCursor = storageCursors.writeCursor(GROUP_CURSOR)) {
                    groupStore.updateRecord(record, groupStoreCursor, NULL, storageCursors);
                }
            }
        }
        storageEngine.flushAndForce(NULL);
    }
    // when
    directory.getFileSystem().deleteFile(layout.relationshipGroupDegreesStore());
    rebuildAndVerify(layout, config, expectedDegrees);
}

Example 29 with RecordDatabaseLayout

use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.

the class IdGeneratorMigratorTest method shouldFindAndStoreDeletedIds.

@Test
void shouldFindAndStoreDeletedIds() throws IOException {
    // given
    RecordDatabaseLayout db = RecordDatabaseLayout.ofFlat(directory.directory("from"));
    RecordDatabaseLayout upgrade = RecordDatabaseLayout.ofFlat(directory.directory("to"));
    long nodeStoreStartId;
    long stringStoreStartId;
    long relationshipStoreStartId;
    try (NeoStores neoStores = new StoreFactory(db, defaults(), new DefaultIdGeneratorFactory(fs, immediate(), DEFAULT_DATABASE_NAME), pageCache, fs, StandardV3_4.RECORD_FORMATS, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable(), immutable.empty()).openAllNeoStores(true)) {
        // Let nodes have every fourth a deleted record
        createSomeRecordsAndSomeHoles(neoStores.getNodeStore(), 500, 1, 3);
        createSomeRecordsAndSomeHoles(neoStores.getPropertyStore().getStringStore(), 100, 1, 2);
        nodeStoreStartId = neoStores.getNodeStore().getNumberOfReservedLowIds();
        stringStoreStartId = neoStores.getPropertyStore().getStringStore().getNumberOfReservedLowIds();
    }
    // Pretend that the relationship store was copied so that relationship id file should be migrated from there
    try (NeoStores neoStores = new StoreFactory(upgrade, defaults(), new DefaultIdGeneratorFactory(fs, immediate(), DEFAULT_DATABASE_NAME), pageCache, fs, Standard.LATEST_RECORD_FORMATS, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable(), immutable.empty()).openAllNeoStores(true)) {
        // Let relationships have every fourth a created record
        createSomeRecordsAndSomeHoles(neoStores.getRelationshipStore(), 600, 3, 1);
        relationshipStoreStartId = neoStores.getRelationshipStore().getNumberOfReservedLowIds();
    }
    fs.deleteFile(upgrade.nodeStore());
    fs.deleteFile(upgrade.idNodeStore());
    fs.deleteFile(upgrade.propertyStringStore());
    fs.deleteFile(upgrade.idPropertyStringStore());
    fs.deleteFile(db.relationshipStore());
    fs.deleteFile(upgrade.idRelationshipStore());
    // when
    IdGeneratorMigrator migrator = new IdGeneratorMigrator(fs, pageCache, defaults(), PageCacheTracer.NULL);
    migrator.migrate(db, upgrade, ProgressReporter.SILENT, StandardV3_4.STORE_VERSION, Standard.LATEST_STORE_VERSION, IndexImporterFactory.EMPTY);
    migrator.moveMigratedFiles(upgrade, db, StandardV3_4.STORE_VERSION, Standard.LATEST_STORE_VERSION);
    // then
    assertIdGeneratorContainsIds(db.idNodeStore(), RecordIdType.NODE, 500, 1, 3, nodeStoreStartId);
    assertIdGeneratorContainsIds(db.idPropertyStringStore(), RecordIdType.STRING_BLOCK, 100, 1, 2, stringStoreStartId);
    assertIdGeneratorContainsIds(db.idRelationshipStore(), RecordIdType.RELATIONSHIP, 600, 3, 1, relationshipStoreStartId);
}