use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.
the class ConsistencyCheckWithCorruptGBPTreeIT method multipleCorruptionsInFusionIndex.
@Test
void multipleCorruptionsInFusionIndex() throws Exception {
// Because NATIVE30 provider use Lucene internally we can not use the snapshot from ephemeral file system because
// lucene will not use it to store the files. Therefor we use a default file system together with TestDirectory
// for cleanup.
final DefaultFileSystemAbstraction fs = new DefaultFileSystemAbstraction();
final TestDirectory testDirectory = TestDirectory.testDirectory(fs);
testDirectory.prepareDirectory(ConsistencyCheckWithCorruptGBPTreeIT.class, "multipleCorruptionsInFusionIndex");
try {
final Path neo4jHome = testDirectory.homePath();
dbmsAction(neo4jHome, fs, NATIVE30, db -> {
Label label = Label.label("label2");
indexWithNumberData(db, label);
}, builder -> {
});
RecordDatabaseLayout layout = RecordDatabaseLayout.of(Config.defaults(neo4j_home, neo4jHome));
final Path[] indexFiles = schemaIndexFiles(fs, layout.databaseDirectory(), NATIVE30);
final List<Path> files = corruptIndexes(fs, readOnly(), (tree, inspection) -> {
long leafNode = inspection.getLeafNodes().get(1);
long internalNode = inspection.getInternalNodes().get(0);
tree.unsafe(pageSpecificCorruption(leafNode, GBPTreeCorruption.rightSiblingPointToNonExisting()), CursorContext.NULL);
tree.unsafe(pageSpecificCorruption(internalNode, GBPTreeCorruption.setChild(0, internalNode)), CursorContext.NULL);
}, indexFiles);
assertTrue(files.size() > 0, "Expected number of corrupted files to be more than one.");
ConsistencyCheckService.Result result = runConsistencyCheck(fs, neo4jHome, layout, NullLogProvider.getInstance(), NONE, DEFAULT);
for (Path file : files) {
assertResultContainsMessage(fs, result, "Index will be excluded from further consistency checks. Index file: " + file.toAbsolutePath());
}
} finally {
testDirectory.cleanup();
}
}
use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.
the class RecordStorageEngineFactory method loadSchemaRules.
@Override
public List<SchemaRule> loadSchemaRules(FileSystemAbstraction fs, PageCache pageCache, Config config, DatabaseLayout layout, CursorContext cursorContext) {
RecordDatabaseLayout databaseLayout = convert(layout);
StoreFactory factory = new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, NullLogProvider.getInstance(), PageCacheTracer.NULL, readOnly());
try (NeoStores stores = factory.openNeoStores(false, StoreType.SCHEMA, StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY);
var storeCursors = new CachedStoreCursors(stores, cursorContext)) {
stores.start(cursorContext);
TokenHolders tokenHolders = tokenHoldersForSchemaStore(stores, new ReadOnlyTokenCreator(), storeCursors);
List<SchemaRule> rules = new ArrayList<>();
new SchemaStorage(stores.getSchemaStore(), tokenHolders, () -> KernelVersion.LATEST).getAll(storeCursors).forEach(rules::add);
return rules;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.
the class DegreesRebuildFromStoreTest method skipNotUsedRecordsOnDegreeStoreRebuild.
@Test
void skipNotUsedRecordsOnDegreeStoreRebuild() throws Exception {
// given a dataset containing mixed sparse and dense nodes with relationships in random directions,
// where some chains have been marked as having external degrees
int denseThreshold = dense_node_threshold.defaultValue();
RecordDatabaseLayout layout = RecordDatabaseLayout.ofFlat(directory.homePath());
int[] relationshipTypes;
MutableLongLongMap expectedDegrees = LongLongMaps.mutable.empty();
Config config = config(denseThreshold);
try (Lifespan life = new Lifespan()) {
RecordStorageEngine storageEngine = openStorageEngine(layout, config);
relationshipTypes = createRelationshipTypes(storageEngine);
life.add(storageEngine);
generateData(storageEngine, denseThreshold, relationshipTypes);
storageEngine.relationshipGroupDegreesStore().accept((groupId, direction, degree) -> expectedDegrees.put(combinedKeyOnGroupAndDirection(groupId, direction), degree), NULL);
assertThat(expectedDegrees.isEmpty()).isFalse();
RelationshipGroupStore groupStore = storageEngine.testAccessNeoStores().getRelationshipGroupStore();
try (StoreCursors storageCursors = storageEngine.createStorageCursors(NULL)) {
long highId = groupStore.getHighId();
assertThat(highId).isGreaterThan(1);
for (int i = 10; i < highId; i++) {
RelationshipGroupRecord record = groupStore.getRecordByCursor(i, new RelationshipGroupRecord(i), RecordLoad.ALWAYS, storageCursors.readCursor(GROUP_CURSOR));
expectedDegrees.remove(combinedKeyOnGroupAndDirection(record.getId(), RelationshipDirection.OUTGOING));
expectedDegrees.remove(combinedKeyOnGroupAndDirection(record.getId(), RelationshipDirection.INCOMING));
expectedDegrees.remove(combinedKeyOnGroupAndDirection(record.getId(), RelationshipDirection.LOOP));
record.setInUse(false);
try (var groupStoreCursor = storageCursors.writeCursor(GROUP_CURSOR)) {
groupStore.updateRecord(record, groupStoreCursor, NULL, storageCursors);
}
}
}
storageEngine.flushAndForce(NULL);
}
// when
directory.getFileSystem().deleteFile(layout.relationshipGroupDegreesStore());
rebuildAndVerify(layout, config, expectedDegrees);
}
use of org.neo4j.io.layout.recordstorage.RecordDatabaseLayout in project neo4j by neo4j.
the class IdGeneratorMigratorTest method shouldFindAndStoreDeletedIds.
@Test
void shouldFindAndStoreDeletedIds() throws IOException {
// given
RecordDatabaseLayout db = RecordDatabaseLayout.ofFlat(directory.directory("from"));
RecordDatabaseLayout upgrade = RecordDatabaseLayout.ofFlat(directory.directory("to"));
long nodeStoreStartId;
long stringStoreStartId;
long relationshipStoreStartId;
try (NeoStores neoStores = new StoreFactory(db, defaults(), new DefaultIdGeneratorFactory(fs, immediate(), DEFAULT_DATABASE_NAME), pageCache, fs, StandardV3_4.RECORD_FORMATS, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable(), immutable.empty()).openAllNeoStores(true)) {
// Let nodes have every fourth a deleted record
createSomeRecordsAndSomeHoles(neoStores.getNodeStore(), 500, 1, 3);
createSomeRecordsAndSomeHoles(neoStores.getPropertyStore().getStringStore(), 100, 1, 2);
nodeStoreStartId = neoStores.getNodeStore().getNumberOfReservedLowIds();
stringStoreStartId = neoStores.getPropertyStore().getStringStore().getNumberOfReservedLowIds();
}
// Pretend that the relationship store was copied so that relationship id file should be migrated from there
try (NeoStores neoStores = new StoreFactory(upgrade, defaults(), new DefaultIdGeneratorFactory(fs, immediate(), DEFAULT_DATABASE_NAME), pageCache, fs, Standard.LATEST_RECORD_FORMATS, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable(), immutable.empty()).openAllNeoStores(true)) {
// Let relationships have every fourth a created record
createSomeRecordsAndSomeHoles(neoStores.getRelationshipStore(), 600, 3, 1);
relationshipStoreStartId = neoStores.getRelationshipStore().getNumberOfReservedLowIds();
}
fs.deleteFile(upgrade.nodeStore());
fs.deleteFile(upgrade.idNodeStore());
fs.deleteFile(upgrade.propertyStringStore());
fs.deleteFile(upgrade.idPropertyStringStore());
fs.deleteFile(db.relationshipStore());
fs.deleteFile(upgrade.idRelationshipStore());
// when
IdGeneratorMigrator migrator = new IdGeneratorMigrator(fs, pageCache, defaults(), PageCacheTracer.NULL);
migrator.migrate(db, upgrade, ProgressReporter.SILENT, StandardV3_4.STORE_VERSION, Standard.LATEST_STORE_VERSION, IndexImporterFactory.EMPTY);
migrator.moveMigratedFiles(upgrade, db, StandardV3_4.STORE_VERSION, Standard.LATEST_STORE_VERSION);
// then
assertIdGeneratorContainsIds(db.idNodeStore(), RecordIdType.NODE, 500, 1, 3, nodeStoreStartId);
assertIdGeneratorContainsIds(db.idPropertyStringStore(), RecordIdType.STRING_BLOCK, 100, 1, 2, stringStoreStartId);
assertIdGeneratorContainsIds(db.idRelationshipStore(), RecordIdType.RELATIONSHIP, 600, 3, 1, relationshipStoreStartId);
}
Aggregations