Search in sources :

Example 1 with StoreMigratorCheckPointer

use of org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer in project neo4j by neo4j.

the class StoreMigrator method moveMigratedFiles.

@Override
public void moveMigratedFiles(File migrationDir, File storeDir, String versionToUpgradeFrom, String versionToUpgradeTo) throws IOException {
    // Move the migrated ones into the store directory
    StoreFile.fileOperation(MOVE, fileSystem, migrationDir, storeDir, StoreFile.currentStoreFiles(), // allow to skip non existent source files
    true, // allow to overwrite target files
    ExistingTargetStrategy.OVERWRITE, StoreFileType.values());
    // move the files with the page cache.
    try {
        Iterable<FileHandle> fileHandles = pageCache.streamFilesRecursive(migrationDir)::iterator;
        for (FileHandle fh : fileHandles) {
            Predicate<StoreFile> predicate = storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fh.getFile().getName());
            if (StreamSupport.stream(StoreFile.currentStoreFiles().spliterator(), false).anyMatch(predicate)) {
                final Optional<PagedFile> optionalPagedFile = pageCache.getExistingMapping(fh.getFile());
                if (optionalPagedFile.isPresent()) {
                    optionalPagedFile.get().close();
                }
                fh.rename(new File(storeDir, fh.getFile().getName()), StandardCopyOption.REPLACE_EXISTING);
            }
        }
    } catch (NoSuchFileException e) {
    //This means that we had no files only present in the page cache, this is fine.
    }
    RecordFormats oldFormat = selectForVersion(versionToUpgradeFrom);
    RecordFormats newFormat = selectForVersion(versionToUpgradeTo);
    boolean movingAwayFromVersionTrailers = oldFormat.hasCapability(VERSION_TRAILERS) && !newFormat.hasCapability(VERSION_TRAILERS);
    if (movingAwayFromVersionTrailers) {
        StoreFile.removeTrailers(versionToUpgradeFrom, fileSystem, storeDir, pageCache.pageSize());
    }
    File neoStore = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    long logVersion = MetaDataStore.getRecord(pageCache, neoStore, Position.LOG_VERSION);
    long lastCommittedTx = MetaDataStore.getRecord(pageCache, neoStore, Position.LAST_TRANSACTION_ID);
    // update or add upgrade id and time and other necessary neostore records
    updateOrAddNeoStoreFieldsAsPartOfMigration(migrationDir, storeDir, versionToUpgradeTo);
    // delete old logs
    legacyLogs.deleteUnusedLogFiles(storeDir);
    if (movingAwayFromVersionTrailers) {
        // write a check point in the log in order to make recovery work in the newer version
        new StoreMigratorCheckPointer(storeDir, fileSystem).checkPoint(logVersion, lastCommittedTx);
    }
}
Also used : PropertyDeduplicator(org.neo4j.kernel.impl.storemigration.legacystore.v21.propertydeduplication.PropertyDeduplicator) Arrays(java.util.Arrays) PageCursor(org.neo4j.io.pagecache.PageCursor) NullLogProvider(org.neo4j.logging.NullLogProvider) StringUtils(org.apache.commons.lang3.StringUtils) UNKNOWN_TX_COMMIT_TIMESTAMP(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP) COPY(org.neo4j.kernel.impl.storemigration.FileOperation.COPY) SchemaIndexProvider(org.neo4j.kernel.api.index.SchemaIndexProvider) CountsComputer(org.neo4j.kernel.impl.store.CountsComputer) Arrays.asList(java.util.Arrays.asList) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) Position(org.neo4j.kernel.impl.store.MetaDataStore.Position) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) RecordCursors(org.neo4j.kernel.impl.store.RecordCursors) SilentMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor) StandardCharsets(java.nio.charset.StandardCharsets) MetaDataRecordFormat(org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat) DELETE(org.neo4j.kernel.impl.storemigration.FileOperation.DELETE) Stream(java.util.stream.Stream) ExecutionSupervisors.withDynamicProcessorAssignment(org.neo4j.unsafe.impl.batchimport.staging.ExecutionSupervisors.withDynamicProcessorAssignment) StoreType(org.neo4j.kernel.impl.store.StoreType) BatchImporter(org.neo4j.unsafe.impl.batchimport.BatchImporter) NodeStore(org.neo4j.kernel.impl.store.NodeStore) StoreFactory(org.neo4j.kernel.impl.store.StoreFactory) Collectors(org.neo4j.unsafe.impl.batchimport.input.Collectors) PagedFile(org.neo4j.io.pagecache.PagedFile) VERSION_TRAILERS(org.neo4j.kernel.impl.store.format.Capability.VERSION_TRAILERS) FormatFamily(org.neo4j.kernel.impl.store.format.FormatFamily) Supplier(java.util.function.Supplier) UNKNOWN_TX_CHECKSUM(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.UNKNOWN_TX_CHECKSUM) BufferedOutputStream(java.io.BufferedOutputStream) StandardCopyOption(java.nio.file.StandardCopyOption) ArrayList(java.util.ArrayList) TransactionId(org.neo4j.kernel.impl.store.TransactionId) StandardV2_2(org.neo4j.kernel.impl.store.format.standard.StandardV2_2) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) StandardV2_1(org.neo4j.kernel.impl.store.format.standard.StandardV2_1) RelationshipStore(org.neo4j.kernel.impl.store.RelationshipStore) StandardV2_0(org.neo4j.kernel.impl.store.format.standard.StandardV2_0) IdMappers(org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers) BiConsumer(java.util.function.BiConsumer) IdGenerators(org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators) StreamSupport(java.util.stream.StreamSupport) DirectRecordStoreMigrator(org.neo4j.kernel.impl.storemigration.DirectRecordStoreMigrator) CoarseBoundedProgressExecutionMonitor(org.neo4j.unsafe.impl.batchimport.staging.CoarseBoundedProgressExecutionMonitor) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) FileOutputStream(java.io.FileOutputStream) RecordFormatSelector.selectForVersion(org.neo4j.kernel.impl.store.format.RecordFormatSelector.selectForVersion) IOException(java.io.IOException) NodeRecordFormat(org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat) LegacyLogs(org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogs) File(java.io.File) Iterables(org.neo4j.helpers.collection.Iterables) ParallelBatchImporter(org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter) NodeRecord(org.neo4j.kernel.impl.store.record.NodeRecord) Configuration(org.neo4j.unsafe.impl.batchimport.Configuration) BufferedReader(java.io.BufferedReader) MigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor) RelationshipRecord(org.neo4j.kernel.impl.store.record.RelationshipRecord) StoreUpgrader(org.neo4j.kernel.impl.storemigration.StoreUpgrader) NoSuchFileException(java.nio.file.NoSuchFileException) BASE_TX_LOG_VERSION(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_VERSION) BASE_TX_CHECKSUM(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_CHECKSUM) PageCache(org.neo4j.io.pagecache.PageCache) InputNode(org.neo4j.unsafe.impl.batchimport.input.InputNode) Predicate(java.util.function.Predicate) Collection(java.util.Collection) StandardOpenOption(java.nio.file.StandardOpenOption) MOVE(org.neo4j.kernel.impl.storemigration.FileOperation.MOVE) LogService(org.neo4j.kernel.impl.logging.LogService) List(java.util.List) StoreFileType(org.neo4j.kernel.impl.storemigration.StoreFileType) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) Writer(java.io.Writer) Optional(java.util.Optional) Inputs(org.neo4j.unsafe.impl.batchimport.input.Inputs) InputIterable(org.neo4j.unsafe.impl.batchimport.InputIterable) InputRelationship(org.neo4j.unsafe.impl.batchimport.input.InputRelationship) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) InputEntity(org.neo4j.unsafe.impl.batchimport.input.InputEntity) DEFAULT_NAME(org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) CustomIOConfigValidator(org.neo4j.kernel.impl.util.CustomIOConfigValidator) AdditionalInitialIds(org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds) NeoStores(org.neo4j.kernel.impl.store.NeoStores) StorePropertyCursor(org.neo4j.kernel.impl.api.store.StorePropertyCursor) CountsTracker(org.neo4j.kernel.impl.store.counts.CountsTracker) ExistingTargetStrategy(org.neo4j.kernel.impl.storemigration.ExistingTargetStrategy) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) RelationshipRecordFormat(org.neo4j.kernel.impl.store.format.standard.RelationshipRecordFormat) OutputStream(java.io.OutputStream) Config(org.neo4j.kernel.configuration.Config) ReadOnlyIdGeneratorFactory(org.neo4j.kernel.impl.store.id.ReadOnlyIdGeneratorFactory) LockService(org.neo4j.kernel.impl.locking.LockService) FileHandle(org.neo4j.io.pagecache.FileHandle) CapabilityType(org.neo4j.kernel.impl.store.format.CapabilityType) PrimitiveRecord(org.neo4j.kernel.impl.store.record.PrimitiveRecord) FIELD_NOT_PRESENT(org.neo4j.kernel.impl.store.format.standard.MetaDataRecordFormat.FIELD_NOT_PRESENT) StoreMigratorCheckPointer(org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer) ExecutionMonitor(org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor) BASE_TX_LOG_BYTE_OFFSET(org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_LOG_BYTE_OFFSET) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) PagedFile(org.neo4j.io.pagecache.PagedFile) FileHandle(org.neo4j.io.pagecache.FileHandle) NoSuchFileException(java.nio.file.NoSuchFileException) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) PagedFile(org.neo4j.io.pagecache.PagedFile) StoreFile(org.neo4j.kernel.impl.storemigration.StoreFile) File(java.io.File) StoreMigratorCheckPointer(org.neo4j.kernel.impl.storemigration.StoreMigratorCheckPointer)

Aggregations

BufferedOutputStream (java.io.BufferedOutputStream)1 BufferedReader (java.io.BufferedReader)1 File (java.io.File)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 Writer (java.io.Writer)1 StandardCharsets (java.nio.charset.StandardCharsets)1 NoSuchFileException (java.nio.file.NoSuchFileException)1 StandardCopyOption (java.nio.file.StandardCopyOption)1 StandardOpenOption (java.nio.file.StandardOpenOption)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Arrays.asList (java.util.Arrays.asList)1 Collection (java.util.Collection)1 List (java.util.List)1 Optional (java.util.Optional)1 BiConsumer (java.util.function.BiConsumer)1 Predicate (java.util.function.Predicate)1 Supplier (java.util.function.Supplier)1