Search in sources :

Example 1 with VERSION_TRAILERS

use of org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS in project neo4j by neo4j.

the class StoreMigrator method moveMigratedFiles.

@Override
public void moveMigratedFiles(File migrationDir, File storeDir, String versionToUpgradeFrom, String versionToUpgradeTo) throws IOException {
    // Move the migrated ones into the store directory
    StoreFile.fileOperation(MOVE, fileSystem, migrationDir, storeDir, StoreFile.currentStoreFiles(), // allow to skip non existent source files
    true, // allow to overwrite target files
    ExistingTargetStrategy.OVERWRITE, StoreFileType.values());
    // move the files with the page cache.
    try {
        Iterable<FileHandle> fileHandles = pageCache.streamFilesRecursive(migrationDir)::iterator;
        for (FileHandle fh : fileHandles) {
            Predicate<StoreFile> predicate = storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fh.getFile().getName());
            if (StreamSupport.stream(StoreFile.currentStoreFiles().spliterator(), false).anyMatch(predicate)) {
                final Optional<PagedFile> optionalPagedFile = pageCache.getExistingMapping(fh.getFile());
                if (optionalPagedFile.isPresent()) {
                    optionalPagedFile.get().close();
                }
                fh.rename(new File(storeDir, fh.getFile().getName()), StandardCopyOption.REPLACE_EXISTING);
            }
        }
    } catch (NoSuchFileException e) {
    //This means that we had no files only present in the page cache, this is fine.
    }
    RecordFormats oldFormat = selectForVersion(versionToUpgradeFrom);
    RecordFormats newFormat = selectForVersion(versionToUpgradeTo);
    boolean movingAwayFromVersionTrailers = oldFormat.hasCapability(VERSION_TRAILERS) && !newFormat.hasCapability(VERSION_TRAILERS);
    if (movingAwayFromVersionTrailers) {
        StoreFile.removeTrailers(versionToUpgradeFrom, fileSystem, storeDir, pageCache.pageSize());
    }
    File neoStore = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    long logVersion = MetaDataStore.getRecord(pageCache, neoStore, Position.LOG_VERSION);
    long lastCommittedTx = MetaDataStore.getRecord(pageCache, neoStore, Position.LAST_TRANSACTION_ID);
    // update or add upgrade id and time and other necessary neostore records
    updateOrAddNeoStoreFieldsAsPartOfMigration(migrationDir, storeDir, versionToUpgradeTo);
    // delete old logs
    legacyLogs.deleteUnusedLogFiles(storeDir);
    if (movingAwayFromVersionTrailers) {
        // write a check point in the log in order to make recovery work in the newer version
        new StoreMigratorCheckPointer(storeDir, fileSystem).checkPoint(logVersion, lastCommittedTx);
    }
}
Also used : PropertyDeduplicator(org.neo4j.kernel.impl.storemigration.legacystore.v21.propertydeduplication.PropertyDeduplicator) Arrays(java.util.Arrays) PageCursor(org.neo4j.io.pagecache.PageCursor) NullLogProvider(org.neo4j.logging.NullLogProvider) StringUtils(org.apache.commons.lang3.StringUtils) UNKNOWN_TX_COMMIT_TIMESTAMP(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP) COPY(org.neo4j.kernel.impl.storemigration.FileOperation.COPY) SchemaIndexProvider(org.neo4j.kernel.api.index.SchemaIndexProvider) CountsComputer(org.neo4j.kernel.impl.store.CountsComputer) Arrays.asList(java.util.Arrays.asList) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) Position(org.neo4j.kernel.impl.store.MetaDataStore.Position) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) RecordCursors(org.neo4j.kernel.impl.store.RecordCursors) SilentMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor) StandardCharsets(java.nio.charset.StandardCharsets) MetaDataRecordFormat(org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat) DELETE(org.neo4j.kernel.impl.storemigration.FileOperation.DELETE) Stream(java.util.stream.Stream) ExecutionSupervisors.withDynamicProcessorAssignment(org.neo4j.unsafe.impl.batchimport.staging.ExecutionSupervisors.withDynamicProcessorAssignment) StoreType(org.neo4j.kernel.impl.store.StoreType) BatchImporter(org.neo4j.unsafe.impl.batchimport.BatchImporter) NodeStore(org.neo4j.kernel.impl.store.NodeStore) StoreFactory(org.neo4j.kernel.impl.store.StoreFactory) Collectors(org.neo4j.unsafe.impl.batchimport.input.Collectors) PagedFile(org.neo4j.io.pagecache.PagedFile) VERSION_TRAILERS(org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS) FormatFamily(org.neo4j.kernel.impl.store.format.FormatFamily) Supplier(java.util.function.Supplier) UNKNOWN_TX_CHECKSUM(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_CHECKSUM) BufferedOutputStream(java.io.BufferedOutputStream) StandardCopyOption(java.nio.file.StandardCopyOption) ArrayList(java.util.ArrayList) TransactionId(org.neo4j.kernel.impl.store.TransactionId) StandardV2_2(org.neo4j.kernel.impl.store.format.standard.StandardV2_2) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) StandardV2_1(org.neo4j.kernel.impl.store.format.standard.StandardV2_1) RelationshipStore(org.neo4j.kernel.impl.store.RelationshipStore) StandardV2_0(org.neo4j.kernel.impl.store.format.standard.StandardV2_0) IdMappers(org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers) BiConsumer(java.util.function.BiConsumer) IdGenerators(org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators) StreamSupport(java.util.stream.StreamSupport) DirectRecordStoreMigrator(org.neo4j.kernel.impl.storemigration.DirectRecordStoreMigrator) CoarseBoundedProgressExecutionMonitor(org.neo4j.unsafe.impl.batchimport.staging.CoarseBoundedProgressExecutionMonitor) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) FileOutputStream(java.io.FileOutputStream) RecordFormatSelector.selectForVersion(org.neo4j.kernel.impl.store.format.RecordFormatSelector.selectForVersion) IOException(java.io.IOException) NodeRecordFormat(org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat) LegacyLogs(org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogs) File(java.io.File) Iterables(org.neo4j.helpers.collection.Iterables) ParallelBatchImporter(org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter) NodeRecord(org.neo4j.kernel.impl.store.record.NodeRecord) Configuration(org.neo4j.unsafe.impl.batchimport.Configuration) BufferedReader(java.io.BufferedReader) MigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor) RelationshipRecord(org.neo4j.kernel.impl.store.record.RelationshipRecord) StoreUpgrader(org.neo4j.kernel.impl.storemigration.StoreUpgrader) NoSuchFileException(java.nio.file.NoSuchFileException) BASE_TX_LOG_VERSION(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_VERSION) BASE_TX_CHECKSUM(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_CHECKSUM) PageCache(org.neo4j.io.pagecache.PageCache) InputNode(org.neo4j.unsafe.impl.batchimport.input.InputNode) Predicate(java.util.function.Predicate) Collection(java.util.Collection) StandardOpenOption(java.nio.file.StandardOpenOption) MOVE(org.neo4j.kernel.impl.storemigration.FileOperation.MOVE) LogService(org.neo4j.kernel.impl.logging.LogService) List(java.util.List) StoreFileType(org.neo4j.kernel.impl.storemigration.StoreFileType) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) Writer(java.io.Writer) Optional(java.util.Optional) Inputs(org.neo4j.unsafe.impl.batchimport.input.Inputs) InputIterable(org.neo4j.unsafe.impl.batchimport.InputIterable) InputRelationship(org.neo4j.unsafe.impl.batchimport.input.InputRelationship) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) InputEntity(org.neo4j.unsafe.impl.batchimport.input.InputEntity) DEFAULT_NAME(org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) CustomIOConfigValidator(org.neo4j.kernel.impl.util.CustomIOConfigValidator) AdditionalInitialIds(org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds) NeoStores(org.neo4j.kernel.impl.store.NeoStores) StorePropertyCursor(org.neo4j.kernel.impl.api.store.StorePropertyCursor) CountsTracker(org.neo4j.kernel.impl.store.counts.CountsTracker) ExistingTargetStrategy(org.neo4j.kernel.impl.storemigration.ExistingTargetStrategy) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) RelationshipRecordFormat(org.neo4j.kernel.impl.store.format.standard.RelationshipRecordFormat) OutputStream(java.io.OutputStream) Config(org.neo4j.kernel.configuration.Config) ReadOnlyIdGeneratorFactory(org.neo4j.kernel.impl.store.id.ReadOnlyIdGeneratorFactory) LockService(org.neo4j.kernel.impl.locking.LockService) FileHandle(org.neo4j.io.pagecache.FileHandle) CapabilityType(org.neo4j.kernel.impl.store.format.CapabilityType) PrimitiveRecord(org.neo4j.kernel.impl.store.record.PrimitiveRecord) FIELD_NOT_PRESENT(org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat.FIELD_NOT_PRESENT) StoreMigratorCheckPointer(org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer) ExecutionMonitor(org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor) BASE_TX_LOG_BYTE_OFFSET(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_BYTE_OFFSET) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) PagedFile(org.neo4j.io.pagecache.PagedFile) FileHandle(org.neo4j.io.pagecache.FileHandle) NoSuchFileException(java.nio.file.NoSuchFileException) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) PagedFile(org.neo4j.io.pagecache.PagedFile) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) File(java.io.File) StoreMigratorCheckPointer(org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer)

Aggregations

BufferedOutputStream (java.io.BufferedOutputStream)1 BufferedReader (java.io.BufferedReader)1 File (java.io.File)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 Writer (java.io.Writer)1 StandardCharsets (java.nio.charset.StandardCharsets)1 NoSuchFileException (java.nio.file.NoSuchFileException)1 StandardCopyOption (java.nio.file.StandardCopyOption)1 StandardOpenOption (java.nio.file.StandardOpenOption)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Arrays.asList (java.util.Arrays.asList)1 Collection (java.util.Collection)1 List (java.util.List)1 Optional (java.util.Optional)1 BiConsumer (java.util.function.BiConsumer)1 Predicate (java.util.function.Predicate)1 Supplier (java.util.function.Supplier)1