Examples with DatabaseLayout org.neo4j.io.layout.DatabaseLayout used on opensource projects

Example 91 with DatabaseLayout

use of org.neo4j.io.layout.DatabaseLayout in project neo4j by neo4j.

the class PushToCloudCommand method fullSize.

private long fullSize(ExecutionContext ctx, NormalizedDatabaseName database) {
    Path configFile = ctx.confDir().resolve(Config.DEFAULT_CONFIG_FILE_NAME);
    DatabaseLayout layout = Neo4jLayout.of(getConfig(configFile)).databaseLayout(database.name());
    long storeFilesSize = FileUtils.sizeOf(layout.databaseDirectory().toFile());
    long txLogSize = readTxLogsSize(layout.getTransactionLogsDirectory());
    long size = txLogSize + storeFilesSize;
    verbose("Determined FullSize=%d bytes from storeFileSize=%d + txLogSize=%d in database '%s'\n", size, storeFilesSize, txLogSize, database.name());
    return size;
}

Example 92 with DatabaseLayout

use of org.neo4j.io.layout.DatabaseLayout in project neo4j by neo4j.

the class IdGeneratorMigratorTest method shouldFindAndStoreDeletedIds.

@Test
void shouldFindAndStoreDeletedIds() throws IOException {
    // given
    DatabaseLayout db = DatabaseLayout.ofFlat(directory.directory("from"));
    DatabaseLayout upgrade = DatabaseLayout.ofFlat(directory.directory("to"));
    long nodeStoreStartId;
    long stringStoreStartId;
    long relationshipStoreStartId;
    try (NeoStores neoStores = new StoreFactory(db, defaults(), new DefaultIdGeneratorFactory(fs, immediate(), DEFAULT_DATABASE_NAME), pageCache, fs, StandardV3_4.RECORD_FORMATS, nullLogProvider(), PageCacheTracer.NULL, writable(), immutable.empty()).openAllNeoStores(true)) {
        // Let nodes have every fourth a deleted record
        createSomeRecordsAndSomeHoles(neoStores.getNodeStore(), 500, 1, 3);
        createSomeRecordsAndSomeHoles(neoStores.getPropertyStore().getStringStore(), 100, 1, 2);
        nodeStoreStartId = neoStores.getNodeStore().getNumberOfReservedLowIds();
        stringStoreStartId = neoStores.getPropertyStore().getStringStore().getNumberOfReservedLowIds();
    }
    // Pretend that the relationship store was copied so that relationship id file should be migrated from there
    try (NeoStores neoStores = new StoreFactory(upgrade, defaults(), new DefaultIdGeneratorFactory(fs, immediate(), DEFAULT_DATABASE_NAME), pageCache, fs, Standard.LATEST_RECORD_FORMATS, nullLogProvider(), PageCacheTracer.NULL, writable(), immutable.empty()).openAllNeoStores(true)) {
        // Let relationships have every fourth a created record
        createSomeRecordsAndSomeHoles(neoStores.getRelationshipStore(), 600, 3, 1);
        relationshipStoreStartId = neoStores.getRelationshipStore().getNumberOfReservedLowIds();
    }
    fs.deleteFile(upgrade.nodeStore());
    fs.deleteFile(upgrade.idNodeStore());
    fs.deleteFile(upgrade.propertyStringStore());
    fs.deleteFile(upgrade.idPropertyStringStore());
    fs.deleteFile(db.relationshipStore());
    fs.deleteFile(upgrade.idRelationshipStore());
    // when
    IdGeneratorMigrator migrator = new IdGeneratorMigrator(fs, pageCache, defaults(), PageCacheTracer.NULL);
    migrator.migrate(db, upgrade, ProgressReporter.SILENT, StandardV3_4.STORE_VERSION, Standard.LATEST_STORE_VERSION, IndexImporterFactory.EMPTY);
    migrator.moveMigratedFiles(upgrade, db, StandardV3_4.STORE_VERSION, Standard.LATEST_STORE_VERSION);
    // then
    assertIdGeneratorContainsIds(db.idNodeStore(), IdType.NODE, 500, 1, 3, nodeStoreStartId);
    assertIdGeneratorContainsIds(db.idPropertyStringStore(), IdType.STRING_BLOCK, 100, 1, 2, stringStoreStartId);
    assertIdGeneratorContainsIds(db.idRelationshipStore(), IdType.RELATIONSHIP, 600, 3, 1, relationshipStoreStartId);
}

Example 93 with DatabaseLayout

use of org.neo4j.io.layout.DatabaseLayout in project neo4j by neo4j.

the class StoreUpgraderInterruptionTestIT method shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMovingFiles.

@Test
public void shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMovingFiles() throws IOException, ConsistencyCheckIncompleteException {
    MigrationTestUtils.prepareSampleLegacyDatabase(version, fs, workingDatabaseLayout.databaseDirectory(), prepareDirectory);
    RecordStoreVersionCheck versionCheck = new RecordStoreVersionCheck(fs, pageCache, workingDatabaseLayout, NullLogProvider.getInstance(), Config.defaults(), NULL);
    MigrationProgressMonitor progressMonitor = MigrationProgressMonitor.SILENT;
    LogService logService = NullLogService.getInstance();
    RecordStorageMigrator failingStoreMigrator = new RecordStorageMigrator(fs, pageCache, CONFIG, logService, jobScheduler, NULL, batchImporterFactory, INSTANCE) {

        @Override
        public void moveMigratedFiles(DatabaseLayout migrationLayout, DatabaseLayout directoryLayout, String versionToUpgradeFrom, String versionToMigrateTo) throws IOException {
            super.moveMigratedFiles(migrationLayout, directoryLayout, versionToUpgradeFrom, versionToMigrateTo);
            throw new RuntimeException("This upgrade is failing");
        }
    };
    IdGeneratorMigrator idMigrator = new IdGeneratorMigrator(fs, pageCache, CONFIG, NULL);
    assertTrue(checkNeoStoreHasFormatVersion(versionCheck, baselineFormat));
    try {
        newUpgrader(versionCheck, progressMonitor, createIndexMigrator(), failingStoreMigrator, idMigrator).migrateIfNeeded(workingDatabaseLayout, false);
        fail("Should throw exception");
    } catch (RuntimeException e) {
        assertEquals("This upgrade is failing", e.getMessage());
    }
    RecordStorageMigrator migrator = new RecordStorageMigrator(fs, pageCache, CONFIG, logService, jobScheduler, NULL, batchImporterFactory, INSTANCE);
    newUpgrader(versionCheck, progressMonitor, createIndexMigrator(), migrator, idMigrator).migrateIfNeeded(workingDatabaseLayout, false);
    assertTrue(checkNeoStoreHasFormatVersion(versionCheck, successorFormat));
    pageCache.close();
    // Since consistency checker is in read only mode we need to start/stop db to generate label scan store.
    startStopDatabase(neo4jLayout.homeDirectory());
    assertConsistentStore(workingDatabaseLayout);
}

Example 94 with DatabaseLayout

use of org.neo4j.io.layout.DatabaseLayout in project neo4j by neo4j.

the class ConsistencyCheckWithCorruptGBPTreeIT method multipleCorruptionsInFusionIndex.

@Test
void multipleCorruptionsInFusionIndex() throws Exception {
    // Because NATIVE30 provider use Lucene internally we can not use the snapshot from ephemeral file system because
    // lucene will not use it to store the files. Therefor we use a default file system together with TestDirectory
    // for cleanup.
    final DefaultFileSystemAbstraction fs = new DefaultFileSystemAbstraction();
    final TestDirectory testDirectory = TestDirectory.testDirectory(fs);
    testDirectory.prepareDirectory(ConsistencyCheckWithCorruptGBPTreeIT.class, "multipleCorruptionsInFusionIndex");
    try {
        final Path neo4jHome = testDirectory.homePath();
        dbmsAction(neo4jHome, fs, NATIVE30, db -> {
            Label label = Label.label("label2");
            indexWithNumberData(db, label);
        }, builder -> {
        });
        DatabaseLayout layout = DatabaseLayout.of(Config.defaults(neo4j_home, neo4jHome));
        final Path[] indexFiles = schemaIndexFiles(fs, layout.databaseDirectory(), NATIVE30);
        final List<Path> files = corruptIndexes(fs, readOnly(), (tree, inspection) -> {
            long leafNode = inspection.getLeafNodes().get(1);
            long internalNode = inspection.getInternalNodes().get(0);
            tree.unsafe(pageSpecificCorruption(leafNode, GBPTreeCorruption.rightSiblingPointToNonExisting()), CursorContext.NULL);
            tree.unsafe(pageSpecificCorruption(internalNode, GBPTreeCorruption.setChild(0, internalNode)), CursorContext.NULL);
        }, indexFiles);
        assertTrue(files.size() > 0, "Expected number of corrupted files to be more than one.");
        ConsistencyCheckService.Result result = runConsistencyCheck(fs, neo4jHome, layout, NullLogProvider.getInstance(), NONE, DEFAULT);
        for (Path file : files) {
            assertResultContainsMessage(fs, result, "Index will be excluded from further consistency checks. Index file: " + file.toAbsolutePath());
        }
    } finally {
        testDirectory.cleanup();
    }
}

Example 95 with DatabaseLayout

use of org.neo4j.io.layout.DatabaseLayout in project neo4j by neo4j.

the class CheckConsistencyCommand method execute.

@Override
public void execute() {
    options.warnOnUsageOfDeprecatedOptions(spec, ctx);
    if (target.backup != null) {
        target.backup = target.backup.toAbsolutePath();
        if (!Files.isDirectory(target.backup)) {
            throw new CommandFailedException("Report directory path doesn't exist or not a directory: " + target.backup);
        }
    }
    Config config = loadNeo4jConfig(ctx.homeDir(), ctx.confDir(), additionalConfig);
    var memoryTracker = EmptyMemoryTracker.INSTANCE;
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction()) {
        DatabaseLayout databaseLayout = Optional.ofNullable(target.backup).map(DatabaseLayout::ofFlat).orElseGet(() -> Neo4jLayout.of(config).databaseLayout(target.database.name()));
        checkDatabaseExistence(databaseLayout);
        try (Closeable ignored = LockChecker.checkDatabaseLock(databaseLayout)) {
            checkDbState(databaseLayout, config, memoryTracker);
            // Only output progress indicator if a console receives the output
            ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.NONE;
            if (System.console() != null) {
                progressMonitorFactory = ProgressMonitorFactory.textual(System.out);
            }
            ConsistencyCheckService.Result consistencyCheckResult;
            try (Log4jLogProvider logProvider = Util.configuredLogProvider(config, System.out)) {
                consistencyCheckResult = consistencyCheckService.runFullConsistencyCheck(databaseLayout, config, progressMonitorFactory, logProvider, fileSystem, verbose, options.getReportDir().normalize(), new ConsistencyFlags(options.isCheckGraph(), options.isCheckIndexes(), options.isCheckIndexStructure()));
            }
            if (!consistencyCheckResult.isSuccessful()) {
                throw new CommandFailedException(format("Inconsistencies found. See '%s' for details.", consistencyCheckResult.reportFile()));
            }
        } catch (FileLockException e) {
            throw new CommandFailedException("The database is in use. Stop database '" + databaseLayout.getDatabaseName() + "' and try again.", e);
        } catch (CannotWriteException e) {
            throw new CommandFailedException("You do not have permission to check database consistency.", e);
        }
    } catch (ConsistencyCheckIncompleteException | IOException e) {
        throw new CommandFailedException("Consistency checking failed." + e.getMessage(), e);
    }
}