Examples with LogFile org.neo4j.kernel.impl.transaction.log.files.LogFile used on opensource projects

Example 31 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class CorruptedLogsTruncatorTest method doNotTruncateLogWithPreAllocatedZeros.

@Test
void doNotTruncateLogWithPreAllocatedZeros() throws IOException {
    life.start();
    generateTransactionLogFiles(logFiles);
    var logFile = logFiles.getLogFile();
    long highestLogVersion = logFile.getHighestLogVersion();
    long fileSizeBeforeAppend = Files.size(logFile.getHighestLogFile());
    LogPosition endOfLogsPosition = new LogPosition(highestLogVersion, fileSizeBeforeAppend);
    FlushablePositionAwareChecksumChannel channel = logFile.getTransactionLogWriter().getChannel();
    for (int i = 0; i < RandomUtils.nextInt(100, 10240); i++) {
        channel.putLong(0);
    }
    channel.prepareForFlush().flush();
    long fileAfterZeroAppend = Files.size(logFile.getHighestLogFile());
    assertNotEquals(fileSizeBeforeAppend, fileAfterZeroAppend);
    logPruner.truncate(endOfLogsPosition);
    assertEquals(TOTAL_NUMBER_OF_LOG_FILES, logFiles.logFiles().length);
    assertEquals(fileAfterZeroAppend, Files.size(logFile.getHighestLogFile()));
    assertNotEquals(fileSizeBeforeAppend, Files.size(logFile.getHighestLogFile()));
    assertTrue(ArrayUtils.isEmpty(databaseDirectory.toFile().listFiles(File::isDirectory)));
}

Example 32 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class TransactionRangeDiagnosticsTest method logs.

private static LogFiles logs(ThrowingConsumer<LogFile, IOException> transactionLogs, ThrowingConsumer<CheckpointFile, IOException> checkpointLogs) throws IOException {
    LogFiles files = mock(TransactionLogFiles.class);
    when(files.logFilesDirectory()).thenReturn(Path.of("."));
    LogFile transactionFiles = mock(LogFile.class);
    when(files.getLogFile()).thenReturn(transactionFiles);
    transactionLogs.accept(transactionFiles);
    CheckpointFile checkpointFiles = mock(CheckpointFile.class);
    when(files.getCheckpointFile()).thenReturn(checkpointFiles);
    checkpointLogs.accept(checkpointFiles);
    return files;
}

Example 33 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class InlinedLogTailScanner method findLogTail.

protected LogTailInformation findLogTail() throws IOException {
    LogFile logFile = logFiles.getLogFile();
    final long highestLogVersion = logFile.getHighestLogVersion();
    long version = highestLogVersion;
    long versionToSearchForCommits = highestLogVersion;
    LogEntryStart latestStartEntry = null;
    long oldestStartEntryTransaction = NO_TRANSACTION_ID;
    long oldestVersionFound = -1;
    byte latestLogEntryVersion = 0;
    boolean startRecordAfterCheckpoint = false;
    boolean corruptedTransactionLogs = false;
    while (version >= logFile.getLowestLogVersion() && version >= INITIAL_LOG_VERSION) {
        log.info("Scanning log file with version %d for checkpoint entries", version);
        oldestVersionFound = version;
        CheckpointInfo latestCheckPoint = null;
        StoreId storeId = StoreId.UNKNOWN;
        try (LogVersionedStoreChannel channel = logFile.openForVersion(version);
            var readAheadChannel = new ReadAheadLogChannel(channel, memoryTracker);
            LogEntryCursor cursor = new LogEntryCursor(logEntryReader, readAheadChannel)) {
            LogHeader logHeader = logFile.extractHeader(version);
            storeId = logHeader.getStoreId();
            LogEntry entry;
            long position = logHeader.getStartPosition().getByteOffset();
            long channelVersion = version;
            while (cursor.next()) {
                entry = cursor.get();
                // Collect data about latest checkpoint
                if (entry instanceof LogEntryInlinedCheckPoint) {
                    latestCheckPoint = new CheckpointInfo((LogEntryInlinedCheckPoint) entry, storeId, new LogPosition(channelVersion, position));
                } else if (entry instanceof LogEntryCommit) {
                    if (oldestStartEntryTransaction == NO_TRANSACTION_ID) {
                        oldestStartEntryTransaction = ((LogEntryCommit) entry).getTxId();
                    }
                } else if (entry instanceof LogEntryStart) {
                    LogEntryStart startEntry = (LogEntryStart) entry;
                    if (version == versionToSearchForCommits) {
                        latestStartEntry = startEntry;
                    }
                    startRecordAfterCheckpoint = true;
                }
                // Collect data about latest entry version, only in first log file
                if (version == versionToSearchForCommits || latestLogEntryVersion == 0) {
                    latestLogEntryVersion = entry.getVersion().version();
                }
                position = channel.position();
                channelVersion = channel.getVersion();
            }
            verifyReaderPosition(version, logEntryReader.lastPosition());
        } catch (Error | ClosedByInterruptException e) {
            // These should not be parsing errors
            throw e;
        } catch (Throwable t) {
            monitor.corruptedLogFile(version, t);
            if (failOnCorruptedLogFiles) {
                throwUnableToCleanRecover(t);
            }
            corruptedTransactionLogs = true;
        }
        if (latestCheckPoint != null) {
            return checkpointTailInformation(highestLogVersion, latestStartEntry, oldestVersionFound, latestLogEntryVersion, latestCheckPoint, corruptedTransactionLogs, storeId);
        }
        version--;
        // if we have found no commits in the latest log, keep searching in the next one
        if (latestStartEntry == null) {
            versionToSearchForCommits--;
        }
    }
    return new LogTailInformation(corruptedTransactionLogs || startRecordAfterCheckpoint, oldestStartEntryTransaction, oldestVersionFound == UNKNOWN, highestLogVersion, latestLogEntryVersion);
}

Example 34 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class AbstractLogTailScanner method verifyReaderPosition.

protected void verifyReaderPosition(long version, LogPosition logPosition) throws IOException {
    LogFile logFile = logFiles.getLogFile();
    long highestLogVersion = logFile.getHighestLogVersion();
    try (var channel = logFile.openForVersion(version)) {
        verifyLogVersion(version, logPosition);
        long logFileSize = channel.size();
        long channelLeftovers = subtractExact(logFileSize, logPosition.getByteOffset());
        if (channelLeftovers != 0) {
            // channel has more data than entry reader can read. Only one valid case for this kind of situation is
            // pre-allocated log file that has some space left
            // if this log file is not the last one and we have some unreadable bytes in the end its an indication of corrupted log files
            verifyLastFile(highestLogVersion, version, logPosition, logFileSize, channelLeftovers);
            // to double check that even when we encountered end of records position we do not have anything after that
            // we will try to read some data (up to 12K) in advance to check that only zero's are available there
            verifyNoMoreReadableDataAvailable(version, channel, logPosition, channelLeftovers);
        }
    }
}

Example 35 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class ThresholdBasedPruneStrategyTest method shouldDeleteJustWhatTheThresholdSays.

@Test
void shouldDeleteJustWhatTheThresholdSays() throws IOException {
    // Given
    when(threshold.reached(any(), eq(6L), any())).thenReturn(false);
    when(threshold.reached(any(), eq(5L), any())).thenReturn(false);
    when(threshold.reached(any(), eq(4L), any())).thenReturn(false);
    when(threshold.reached(any(), eq(3L), any())).thenReturn(true);
    Path fileName1 = Path.of("logical.log.v1");
    Path fileName2 = Path.of("logical.log.v2");
    Path fileName3 = Path.of("logical.log.v3");
    Path fileName4 = Path.of("logical.log.v4");
    Path fileName5 = Path.of("logical.log.v5");
    Path fileName6 = Path.of("logical.log.v6");
    when(logFile.getLogFileForVersion(6)).thenReturn(fileName6);
    when(logFile.getLogFileForVersion(5)).thenReturn(fileName5);
    when(logFile.getLogFileForVersion(4)).thenReturn(fileName4);
    when(logFile.getLogFileForVersion(3)).thenReturn(fileName3);
    when(logFile.getLogFileForVersion(2)).thenReturn(fileName2);
    when(logFile.getLogFileForVersion(1)).thenReturn(fileName1);
    when(logFile.getLowestLogVersion()).thenReturn(1L);
    when(fileSystem.getFileSize(any(Path.class))).thenReturn(CURRENT_FORMAT_LOG_HEADER_SIZE + 1L);
    ThresholdBasedPruneStrategy strategy = new ThresholdBasedPruneStrategy(logFile, threshold);
    // When
    strategy.findLogVersionsToDelete(7L).forEachOrdered(uncheckedLongConsumer(v -> fileSystem.deleteFile(logFile.getLogFileForVersion(v))));
    // Then
    verify(threshold).init();
    verify(fileSystem).deleteFile(fileName1);
    verify(fileSystem).deleteFile(fileName2);
    verify(fileSystem).deleteFile(fileName3);
    verify(fileSystem, never()).deleteFile(fileName4);
    verify(fileSystem, never()).deleteFile(fileName5);
    verify(fileSystem, never()).deleteFile(fileName6);
}