Examples with RecordFormats org.neo4j.kernel.impl.store.format.RecordFormats used on opensource projects

Example 6 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class SchemaIndexMigrator method migrate.

@Override
public void migrate(File storeDir, File migrationDir, MigrationProgressMonitor.Section progressMonitor, String versionToMigrateFrom, String versionToMigrateTo) throws IOException {
    RecordFormats from = RecordFormatSelector.selectForVersion(versionToMigrateFrom);
    RecordFormats to = RecordFormatSelector.selectForVersion(versionToMigrateTo);
    if (!from.hasSameCapabilities(to, CapabilityType.INDEX)) {
        schemaIndexDirectory = schemaIndexProvider.getSchemaIndexStoreDirectory(storeDir);
        labelIndexDirectory = labelScanStoreProvider.getStoreDirectory(storeDir);
        deleteObsoleteIndexes = true;
    }
}

Example 7 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class StoreMigrator method moveMigratedFiles.

@Override
public void moveMigratedFiles(File migrationDir, File storeDir, String versionToUpgradeFrom, String versionToUpgradeTo) throws IOException {
    // Move the migrated ones into the store directory
    StoreFile.fileOperation(MOVE, fileSystem, migrationDir, storeDir, StoreFile.currentStoreFiles(), // allow to skip non existent source files
    true, // allow to overwrite target files
    ExistingTargetStrategy.OVERWRITE, StoreFileType.values());
    // move the files with the page cache.
    try {
        Iterable<FileHandle> fileHandles = pageCache.streamFilesRecursive(migrationDir)::iterator;
        for (FileHandle fh : fileHandles) {
            Predicate<StoreFile> predicate = storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fh.getFile().getName());
            if (StreamSupport.stream(StoreFile.currentStoreFiles().spliterator(), false).anyMatch(predicate)) {
                final Optional<PagedFile> optionalPagedFile = pageCache.getExistingMapping(fh.getFile());
                if (optionalPagedFile.isPresent()) {
                    optionalPagedFile.get().close();
                }
                fh.rename(new File(storeDir, fh.getFile().getName()), StandardCopyOption.REPLACE_EXISTING);
            }
        }
    } catch (NoSuchFileException e) {
    //This means that we had no files only present in the page cache, this is fine.
    }
    RecordFormats oldFormat = selectForVersion(versionToUpgradeFrom);
    RecordFormats newFormat = selectForVersion(versionToUpgradeTo);
    boolean movingAwayFromVersionTrailers = oldFormat.hasCapability(VERSION_TRAILERS) && !newFormat.hasCapability(VERSION_TRAILERS);
    if (movingAwayFromVersionTrailers) {
        StoreFile.removeTrailers(versionToUpgradeFrom, fileSystem, storeDir, pageCache.pageSize());
    }
    File neoStore = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    long logVersion = MetaDataStore.getRecord(pageCache, neoStore, Position.LOG_VERSION);
    long lastCommittedTx = MetaDataStore.getRecord(pageCache, neoStore, Position.LAST_TRANSACTION_ID);
    // update or add upgrade id and time and other necessary neostore records
    updateOrAddNeoStoreFieldsAsPartOfMigration(migrationDir, storeDir, versionToUpgradeTo);
    // delete old logs
    legacyLogs.deleteUnusedLogFiles(storeDir);
    if (movingAwayFromVersionTrailers) {
        // write a check point in the log in order to make recovery work in the newer version
        new StoreMigratorCheckPointer(storeDir, fileSystem).checkPoint(logVersion, lastCommittedTx);
    }
}

Example 8 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class StoreMigrator method migrate.

@Override
public void migrate(File storeDir, File migrationDir, MigrationProgressMonitor.Section progressMonitor, String versionToMigrateFrom, String versionToMigrateTo) throws IOException {
    if (versionToMigrateFrom.equals(StandardV2_0.STORE_VERSION) || versionToMigrateFrom.equals(StandardV2_1.STORE_VERSION) || versionToMigrateFrom.equals(StandardV2_2.STORE_VERSION)) {
        // These versions are not supported for block devices.
        CustomIOConfigValidator.assertCustomIOConfigNotUsed(config, CUSTOM_IO_EXCEPTION_MESSAGE);
    }
    // Extract information about the last transaction from legacy neostore
    File neoStore = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    long lastTxId = MetaDataStore.getRecord(pageCache, neoStore, Position.LAST_TRANSACTION_ID);
    TransactionId lastTxInfo = extractTransactionIdInformation(neoStore, storeDir, lastTxId);
    LogPosition lastTxLogPosition = extractTransactionLogPosition(neoStore, storeDir, lastTxId);
    // Write the tx checksum to file in migrationDir, because we need it later when moving files into storeDir
    writeLastTxInformation(migrationDir, lastTxInfo);
    writeLastTxLogPosition(migrationDir, lastTxLogPosition);
    if (versionToMigrateFrom.equals("vE.H.0")) {
        // NOTE for 3.0 here is a special case for vE.H.0 "from" record format.
        // Legend has it that 3.0.5 enterprise changed store format without changing store version.
        // This was done to cheat the migrator to avoid doing store migration since the
        // format itself was backwards compatible. Immediately a problem was detected:
        // if a user uses 3.0.5 for a while and then goes back to a previous 3.0.x patch release
        // the db wouldn't recognize it was an incompatible downgrade and start up normally,
        // but read records with scrambled values and pointers, sort of.
        //
        // This condition has two functions:
        //  1. preventing actual store migration between vE.H.0 --> vE.H.0b
        //  2. making vE.H.0b used in any migration where either vE.H.0 or vE.H.0b is the existing format,
        //     this because vE.H.0b is a superset of vE.H.0 and sometimes (for 3.0.5) vE.H.0
        //     actually means vE.H.0b (in later version).
        //
        // In later versions of neo4j there are better mechanics in place so that a non-migration like this
        // can be performed w/o special casing. To not require backporting that functionality
        // this condition is here and should be removed in 3.1.
        versionToMigrateFrom = "vE.H.0b";
    }
    RecordFormats oldFormat = selectForVersion(versionToMigrateFrom);
    RecordFormats newFormat = selectForVersion(versionToMigrateTo);
    if (FormatFamily.isHigherFamilyFormat(newFormat, oldFormat) || (FormatFamily.isSameFamily(oldFormat, newFormat) && isDifferentCapabilities(oldFormat, newFormat))) {
        // TODO if this store has relationship indexes then warn user about that they will be incorrect
        // after migration, because now we're rewriting the relationship ids.
        // Some form of migration is required (a fallback/catch-all option)
        migrateWithBatchImporter(storeDir, migrationDir, lastTxId, lastTxInfo.checksum(), lastTxLogPosition.getLogVersion(), lastTxLogPosition.getByteOffset(), progressMonitor, oldFormat, newFormat);
    }
    if (versionToMigrateFrom.equals(StandardV2_1.STORE_VERSION)) {
        removeDuplicateEntityProperties(storeDir, migrationDir, pageCache, schemaIndexProvider, oldFormat);
    }
// DO NOT migrate logs. LegacyLogs is able to migrate logs, but only changes its format, not any
// contents of it, and since the record format has changed there would be a mismatch between the
// commands in the log and the contents in the store. If log migration is to be performed there
// must be a proper translation happening while doing so.
}

Example 9 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class UpgradableDatabase method checkUpgradeable.

/**
     * Assumed to only be called if {@link #hasCurrentVersion(File)} returns {@code false}.
     *
     * @param storeDirectory the store to check for upgradability is in.
     * @return the {@link RecordFormats} the current store (which is upgradable) is currently in.
     * @throws UpgradeMissingStoreFilesException if store cannot be upgraded due to some store files are missing.
     * @throws UpgradingStoreVersionNotFoundException if store cannot be upgraded due to store
     * version cannot be determined.
     * @throws UnexpectedUpgradingStoreVersionException if store cannot be upgraded due to an unexpected store
     * version found.
     * @throws UnexpectedUpgradingStoreFormatException if store cannot be upgraded due to an unexpected store
     * format found.
     * @throws DatabaseNotCleanlyShutDownException if store cannot be upgraded due to not being cleanly shut down.
     */
public RecordFormats checkUpgradeable(File storeDirectory) {
    Result result = storeVersionCheck.hasVersion(new File(storeDirectory, MetaDataStore.DEFAULT_NAME), format.storeVersion());
    if (result.outcome.isSuccessful()) {
        // Although this method should not have been called in this case.
        return format;
    }
    RecordFormats fromFormat;
    try {
        fromFormat = RecordFormatSelector.selectForVersion(result.actualVersion);
        // of the config setting to change since downgrades aren't possible but the store can still be opened.
        if (FormatFamily.isLowerFamilyFormat(format, fromFormat)) {
            throw new StoreUpgrader.UnexpectedUpgradingStoreFormatException();
        }
        if (FormatFamily.isSameFamily(fromFormat, format) && (fromFormat.generation() > format.generation())) {
            // Tried to downgrade, that isn't supported
            result = new Result(Outcome.attemptedStoreDowngrade, fromFormat.storeVersion(), new File(storeDirectory, MetaDataStore.DEFAULT_NAME).getAbsolutePath());
        } else {
            result = fromFormat.hasCapability(Capability.VERSION_TRAILERS) ? checkCleanShutDownByVersionTrailer(storeDirectory, fromFormat) : checkCleanShutDownByCheckPoint(storeDirectory);
            if (result.outcome.isSuccessful()) {
                return fromFormat;
            }
        }
    } catch (IllegalArgumentException e) {
        result = new Result(Outcome.unexpectedStoreVersion, result.actualVersion, result.storeFilename);
    }
    switch(result.outcome) {
        case missingStoreFile:
            throw new StoreUpgrader.UpgradeMissingStoreFilesException(getPathToStoreFile(storeDirectory, result));
        case storeVersionNotFound:
            throw new StoreUpgrader.UpgradingStoreVersionNotFoundException(getPathToStoreFile(storeDirectory, result));
        case attemptedStoreDowngrade:
            throw new StoreUpgrader.AttemptedDowngradeException();
        case unexpectedStoreVersion:
            throw new StoreUpgrader.UnexpectedUpgradingStoreVersionException(result.actualVersion, format.storeVersion());
        case storeNotCleanlyShutDown:
            throw new StoreUpgrader.DatabaseNotCleanlyShutDownException();
        default:
            throw new IllegalArgumentException("Unexpected outcome: " + result.outcome.name());
    }
}

Example 10 with RecordFormats

use of org.neo4j.kernel.impl.store.format.RecordFormats in project neo4j by neo4j.

the class BatchingNeoStoresTest method shouldNotOpenStoreWithNodesOrRelationshipsInIt.

@Test
public void shouldNotOpenStoreWithNodesOrRelationshipsInIt() throws Exception {
    // GIVEN
    someDataInTheDatabase();
    // WHEN
    try {
        RecordFormats recordFormats = RecordFormatSelector.selectForConfig(Config.empty(), NullLogProvider.getInstance());
        BatchingNeoStores.batchingNeoStores(fsr.get(), storeDir, recordFormats, DEFAULT, NullLogService.getInstance(), EMPTY, Config.empty());
        fail("Should fail on existing data");
    } catch (IllegalStateException e) {
        // THEN
        assertThat(e.getMessage(), containsString("already contains"));
    }
}