Examples with LogPosition org.neo4j.kernel.impl.transaction.log.LogPosition used on opensource projects

Example 11 with LogPosition

use of org.neo4j.kernel.impl.transaction.log.LogPosition in project neo4j by neo4j.

the class VersionAwareLogEntryReader method readLogEntry.

@Override
public LogEntry readLogEntry(SOURCE channel) throws IOException {
    try {
        LogPositionMarker positionMarker = new LogPositionMarker();
        channel.getCurrentPosition(positionMarker);
        while (true) {
            LogEntryVersion version = null;
            LogEntryParser<LogEntry> entryReader;
            try {
                /*
                     * if the read type is negative than it is actually the log entry version
                     * so we need to read an extra byte which will contain the type
                     */
                byte typeCode = channel.get();
                byte versionCode = 0;
                if (typeCode < 0) {
                    versionCode = typeCode;
                    typeCode = channel.get();
                }
                version = byVersion(versionCode);
                entryReader = version.entryParser(typeCode);
            } catch (ReadPastEndException e) {
                // Make these exceptions slip by straight out to the outer handler
                throw e;
            } catch (Exception e) {
                // Tag all other exceptions with log position and other useful information
                LogPosition position = positionMarker.newPosition();
                e = withMessage(e, e.getMessage() + ". At position " + position + " and entry version " + version);
                throw launderedException(IOException.class, e);
            }
            LogEntry entry = entryReader.parse(version, channel, positionMarker, commandReaderFactory);
            if (!entryReader.skip()) {
                return entry;
            }
        }
    } catch (ReadPastEndException e) {
        return null;
    }
}

Example 12 with LogPosition

use of org.neo4j.kernel.impl.transaction.log.LogPosition in project neo4j by neo4j.

the class KernelRecoveryTest method shouldHandleWritesProperlyAfterRecovery.

@Test
public void shouldHandleWritesProperlyAfterRecovery() throws Exception {
    // Given
    EphemeralFileSystemAbstraction fs = fsRule.get();
    GraphDatabaseService db = newDB(fs);
    long node1 = createNode(db);
    // And given the power goes out
    EphemeralFileSystemAbstraction crashedFs = fs.snapshot();
    db.shutdown();
    try {
        db = newDB(crashedFs);
        long node2 = createNode(db);
        db.shutdown();
        // Then the logical log should be in sync
        File logFile = new File(storeDir, PhysicalLogFile.DEFAULT_NAME + PhysicalLogFile.DEFAULT_VERSION_SUFFIX + "0");
        assertThat(logEntries(crashedFs, logFile), containsExactly(// Tx before recovery
        startEntry(-1, -1), commandEntry(node1, NodeCommand.class), commandEntry(ReadOperations.ANY_LABEL, NodeCountsCommand.class), commitEntry(2), // Tx after recovery
        startEntry(-1, -1), commandEntry(node2, NodeCommand.class), commandEntry(ReadOperations.ANY_LABEL, NodeCountsCommand.class), commitEntry(3), // checkpoint
        checkPoint(new LogPosition(0, 250))));
    } finally {
        crashedFs.close();
    }
}

Example 13 with LogPosition

use of org.neo4j.kernel.impl.transaction.log.LogPosition in project neo4j by neo4j.

the class StoreMigrator method migrate.

@Override
public void migrate(File storeDir, File migrationDir, MigrationProgressMonitor.Section progressMonitor, String versionToMigrateFrom, String versionToMigrateTo) throws IOException {
    if (versionToMigrateFrom.equals(StandardV2_0.STORE_VERSION) || versionToMigrateFrom.equals(StandardV2_1.STORE_VERSION) || versionToMigrateFrom.equals(StandardV2_2.STORE_VERSION)) {
        // These versions are not supported for block devices.
        CustomIOConfigValidator.assertCustomIOConfigNotUsed(config, CUSTOM_IO_EXCEPTION_MESSAGE);
    }
    // Extract information about the last transaction from legacy neostore
    File neoStore = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    long lastTxId = MetaDataStore.getRecord(pageCache, neoStore, Position.LAST_TRANSACTION_ID);
    TransactionId lastTxInfo = extractTransactionIdInformation(neoStore, storeDir, lastTxId);
    LogPosition lastTxLogPosition = extractTransactionLogPosition(neoStore, storeDir, lastTxId);
    // Write the tx checksum to file in migrationDir, because we need it later when moving files into storeDir
    writeLastTxInformation(migrationDir, lastTxInfo);
    writeLastTxLogPosition(migrationDir, lastTxLogPosition);
    if (versionToMigrateFrom.equals("vE.H.0")) {
        // NOTE for 3.0 here is a special case for vE.H.0 "from" record format.
        // Legend has it that 3.0.5 enterprise changed store format without changing store version.
        // This was done to cheat the migrator to avoid doing store migration since the
        // format itself was backwards compatible. Immediately a problem was detected:
        // if a user uses 3.0.5 for a while and then goes back to a previous 3.0.x patch release
        // the db wouldn't recognize it was an incompatible downgrade and start up normally,
        // but read records with scrambled values and pointers, sort of.
        //
        // This condition has two functions:
        //  1. preventing actual store migration between vE.H.0 --> vE.H.0b
        //  2. making vE.H.0b used in any migration where either vE.H.0 or vE.H.0b is the existing format,
        //     this because vE.H.0b is a superset of vE.H.0 and sometimes (for 3.0.5) vE.H.0
        //     actually means vE.H.0b (in later version).
        //
        // In later versions of neo4j there are better mechanics in place so that a non-migration like this
        // can be performed w/o special casing. To not require backporting that functionality
        // this condition is here and should be removed in 3.1.
        versionToMigrateFrom = "vE.H.0b";
    }
    RecordFormats oldFormat = selectForVersion(versionToMigrateFrom);
    RecordFormats newFormat = selectForVersion(versionToMigrateTo);
    if (FormatFamily.isHigherFamilyFormat(newFormat, oldFormat) || (FormatFamily.isSameFamily(oldFormat, newFormat) && isDifferentCapabilities(oldFormat, newFormat))) {
        // TODO if this store has relationship indexes then warn user about that they will be incorrect
        // after migration, because now we're rewriting the relationship ids.
        // Some form of migration is required (a fallback/catch-all option)
        migrateWithBatchImporter(storeDir, migrationDir, lastTxId, lastTxInfo.checksum(), lastTxLogPosition.getLogVersion(), lastTxLogPosition.getByteOffset(), progressMonitor, oldFormat, newFormat);
    }
    if (versionToMigrateFrom.equals(StandardV2_1.STORE_VERSION)) {
        removeDuplicateEntityProperties(storeDir, migrationDir, pageCache, schemaIndexProvider, oldFormat);
    }
// DO NOT migrate logs. LegacyLogs is able to migrate logs, but only changes its format, not any
// contents of it, and since the record format has changed there would be a mismatch between the
// commands in the log and the contents in the store. If log migration is to be performed there
// must be a proper translation happening while doing so.
}

Example 14 with LogPosition

use of org.neo4j.kernel.impl.transaction.log.LogPosition in project neo4j by neo4j.

the class StoreMigratorCheckPointer method checkPoint.

/**
     * Write a check point in the log file with the given version
     * <p>
     * It will create the file with header containing the log version and lastCommittedTx given as arguments
     *
     * @param logVersion the log version to open
     * @param lastCommittedTx the last committed tx id
     */
public void checkPoint(long logVersion, long lastCommittedTx) throws IOException {
    PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, fileSystem);
    File logFileForVersion = logFiles.getLogFileForVersion(logVersion);
    if (!fileSystem.fileExists(logFileForVersion)) {
        try (StoreChannel channel = fileSystem.create(logFileForVersion)) {
            writeLogHeader(channel, logVersion, lastCommittedTx);
        }
    }
    try (LogVersionedStoreChannel storeChannel = PhysicalLogFile.openForVersion(logFiles, fileSystem, logVersion, true)) {
        long offset = storeChannel.size();
        storeChannel.position(offset);
        try (PositionAwarePhysicalFlushableChannel channel = new PositionAwarePhysicalFlushableChannel(storeChannel)) {
            TransactionLogWriter writer = new TransactionLogWriter(new LogEntryWriter(channel));
            writer.checkPoint(new LogPosition(logVersion, offset));
        }
    }
}

Example 15 with LogPosition

use of org.neo4j.kernel.impl.transaction.log.LogPosition in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}