Examples with RecordFormats org.neo4j.kernel.impl.store.format.RecordFormats used on opensource projects

Example 26 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class NeoStores method verifyRecordFormat.

private void verifyRecordFormat(StoreType[] storeTypes, CursorContext cursorContext) {
    String expectedStoreVersion = recordFormats.storeVersion();
    long existingFormat;
    if (!fileSystem.fileExists(layout.metadataStore())) {
        // If the meta data store doesn't even exist then look no further, there's nothing to verify
        return;
    }
    if (contains(storeTypes, StoreType.META_DATA)) {
        // We're going to open this store anyway so might as well do it here, like we open the others
        MetaDataStore metaDataStore = (MetaDataStore) getOrOpenStore(StoreType.META_DATA, cursorContext);
        existingFormat = metaDataStore.getRecord(STORE_VERSION.id(), metaDataStore.newRecord(), RecordLoad.CHECK, cursorContext).getValue();
    } else {
        // if we have createIfNotExists set, but don't have the meta data store in the list of stores to open
        try {
            existingFormat = MetaDataStore.getRecord(pageCache, layout.metadataStore(), STORE_VERSION, layout.getDatabaseName(), cursorContext);
        } catch (NoSuchFileException e) {
            // Weird that the file isn't here after we passed the exists check above. Regardless, treat this the same way as above.
            return;
        } catch (IOException e) {
            throw new UnderlyingStorageException(e);
        }
    }
    if (existingFormat != MetaDataRecordFormat.FIELD_NOT_PRESENT) {
        String actualStoreVersion = versionLongToString(existingFormat);
        RecordFormats actualStoreFormat = RecordFormatSelector.selectForVersion(actualStoreVersion);
        if (!isCompatibleFormats(actualStoreFormat)) {
            throw new UnexpectedStoreVersionException(actualStoreVersion, expectedStoreVersion);
        }
    }
}

Example 27 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class DefaultDatabaseManagerUpgradeIT method upgradeDatabase.

@Test
void upgradeDatabase() {
    // Given
    createDbms();
    GraphDatabaseAPI db = (GraphDatabaseAPI) dbms.database(DEFAULT_DATABASE_NAME);
    DefaultDatabaseManager databaseManager = getDatabaseManager(db);
    RecordStoreVersionCheck check = new RecordStoreVersionCheck(fs, getPageCache(db), databaseLayout, NullLogProvider.getInstance(), Config.defaults(), NULL);
    assertFalse(db.isAvailable(100), "Expected database to have failed during startup because we don't allow upgrade.");
    // When
    databaseManager.upgradeDatabase(db.databaseId());
    // Then
    assertTrue(db.isAvailable(100), "Expected database to be available after upgrade");
    RecordFormats expectedFormat = RecordFormatSelector.findLatestFormatInFamily(StandardV3_4.RECORD_FORMATS).orElseThrow();
    assertTrue(MigrationTestUtils.checkNeoStoreHasFormatVersion(check, expectedFormat), "Expected store version to be default.");
}

Example 28 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class NeoStoresTest method newStoreFactory.

private static StoreFactory newStoreFactory(DatabaseLayout databaseLayout, PageCache pageCache, FileSystemAbstraction fs) {
    RecordFormats recordFormats = RecordFormatSelector.defaultFormat();
    Config config = Config.defaults();
    IdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName());
    return new StoreFactory(databaseLayout, config, idGeneratorFactory, pageCache, fs, recordFormats, LOG_PROVIDER, PageCacheTracer.NULL, writable(), immutable.empty());
}

Example 29 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class StoreMigrator method migrateWithBatchImporter.

private void migrateWithBatchImporter(File storeDir, File migrationDir, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, MigrationProgressMonitor.Section progressMonitor, RecordFormats oldFormat, RecordFormats newFormat) throws IOException {
    prepareBatchImportMigration(storeDir, migrationDir, oldFormat, newFormat);
    boolean requiresDynamicStoreMigration = !newFormat.dynamic().equals(oldFormat.dynamic());
    boolean requiresPropertyMigration = !newFormat.property().equals(oldFormat.property()) || requiresDynamicStoreMigration;
    File badFile = new File(storeDir, Configuration.BAD_FILE_NAME);
    try (NeoStores legacyStore = instantiateLegacyStore(oldFormat, storeDir);
        RecordCursors nodeInputCursors = new RecordCursors(legacyStore);
        RecordCursors relationshipInputCursors = new RecordCursors(legacyStore);
        OutputStream badOutput = new BufferedOutputStream(new FileOutputStream(badFile, false))) {
        Configuration importConfig = new Configuration.Overridden(config);
        AdditionalInitialIds additionalInitialIds = readAdditionalIds(lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset);
        // We have to make sure to keep the token ids if we're migrating properties/labels
        BatchImporter importer = new ParallelBatchImporter(migrationDir.getAbsoluteFile(), fileSystem, pageCache, importConfig, logService, withDynamicProcessorAssignment(migrationBatchImporterMonitor(legacyStore, progressMonitor, importConfig), importConfig), additionalInitialIds, config, newFormat);
        InputIterable<InputNode> nodes = legacyNodesAsInput(legacyStore, requiresPropertyMigration, nodeInputCursors);
        InputIterable<InputRelationship> relationships = legacyRelationshipsAsInput(legacyStore, requiresPropertyMigration, relationshipInputCursors);
        importer.doImport(Inputs.input(nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), Collectors.badCollector(badOutput, 0)));
        // During migration the batch importer doesn't necessarily writes all entities, depending on
        // which stores needs migration. Node, relationship, relationship group stores are always written
        // anyways and cannot be avoided with the importer, but delete the store files that weren't written
        // (left empty) so that we don't overwrite those in the real store directory later.
        Collection<StoreFile> storesToDeleteFromMigratedDirectory = new ArrayList<>();
        storesToDeleteFromMigratedDirectory.add(StoreFile.NEO_STORE);
        if (!requiresPropertyMigration) {
            // We didn't migrate properties, so the property stores in the migrated store are just empty/bogus
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.PROPERTY_STORE, StoreFile.PROPERTY_STRING_STORE, StoreFile.PROPERTY_ARRAY_STORE));
        }
        if (!requiresDynamicStoreMigration) {
            // We didn't migrate labels (dynamic node labels) or any other dynamic store
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.NODE_LABEL_STORE, StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, StoreFile.SCHEMA_STORE));
        }
        StoreFile.fileOperation(DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values());
        // When migrating on a block device there might be some files only accessible via the page cache.
        try {
            Predicate<FileHandle> fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream().anyMatch(storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fileHandle.getFile().getName()));
            pageCache.streamFilesRecursive(migrationDir).filter(fileHandlePredicate).forEach(FileHandle.HANDLE_DELETE);
        } catch (NoSuchFileException e) {
        // This means that we had no files only present in the page cache, this is fine.
        }
    }
}

Example 30 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class VersionCommand method execute.

@Override
public void execute(String[] args) throws IncorrectUsage, CommandFailed {
    final Path storeDir = arguments.parseMandatoryPath("store", args);
    Validators.CONTAINS_EXISTING_DATABASE.validate(storeDir.toFile());
    try (DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
        PageCache pageCache = StandalonePageCacheFactory.createPageCache(fileSystem)) {
        final String storeVersion = new StoreVersionCheck(pageCache).getVersion(storeDir.resolve(MetaDataStore.DEFAULT_NAME).toFile()).orElseThrow(() -> new CommandFailed(String.format("Could not find version metadata in store '%s'", storeDir)));
        final String fmt = "%-25s%s";
        out.accept(String.format(fmt, "Store format version:", storeVersion));
        RecordFormats format = RecordFormatSelector.selectForVersion(storeVersion);
        out.accept(String.format(fmt, "Introduced in version:", format.introductionVersion()));
        findSuccessor(format).map(next -> String.format(fmt, "Superseded in version:", next.introductionVersion())).ifPresent(out);
        out.accept(String.format(fmt, "Current version:", Version.getNeo4jVersion()));
    } catch (IOException e) {
        throw new CommandFailed(e.getMessage(), e);
    }
}