Examples with CheckpointFile org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile used on opensource projects

Example 1 with CheckpointFile

use of org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile in project neo4j by neo4j.

the class RecoveryCorruptedTransactionLogIT method restoreCheckpointLogVersionFromFileVersion.

@Test
void restoreCheckpointLogVersionFromFileVersion() throws IOException {
    DatabaseManagementService managementService = databaseFactory.build();
    GraphDatabaseAPI database = (GraphDatabaseAPI) managementService.database(DEFAULT_DATABASE_NAME);
    logFiles = buildDefaultLogFiles(getStoreId(database));
    generateTransaction(database);
    managementService.shutdown();
    int rotations = 10;
    try (Lifespan lifespan = new Lifespan(logFiles)) {
        CheckpointFile checkpointFile = logFiles.getCheckpointFile();
        DetachedCheckpointAppender checkpointAppender = (DetachedCheckpointAppender) checkpointFile.getCheckpointAppender();
        for (int i = 0; i < rotations; i++) {
            checkpointAppender.checkPoint(LogCheckPointEvent.NULL, new LogPosition(0, HEADER_OFFSET), Instant.now(), "test" + i);
            checkpointAppender.rotate();
        }
    }
    for (int i = rotations - 1; i > 0; i--) {
        var restartedDbms = databaseFactory.build();
        try {
            StorageEngine storageEngine = ((GraphDatabaseAPI) restartedDbms.database(DEFAULT_DATABASE_NAME)).getDependencyResolver().resolveDependency(StorageEngine.class);
            assertEquals(i, storageEngine.metadataProvider().getCheckpointLogVersion());
        } finally {
            restartedDbms.shutdown();
        }
        // we remove 3 checkpoints: 1 from shutdown and 1 from recovery and one that we created in a loop before
        removeLastCheckpointRecordFromLastLogFile();
        removeLastCheckpointRecordFromLastLogFile();
        removeLastCheckpointRecordFromLastLogFile();
    }
}

Example 2 with CheckpointFile

use of org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile in project neo4j by neo4j.

the class RecoveryCorruptedTransactionLogIT method removeLastCheckpointRecordFromLastLogFile.

private void removeLastCheckpointRecordFromLastLogFile() throws IOException {
    CheckpointFile checkpointFile = logFiles.getCheckpointFile();
    var checkpoint = checkpointFile.findLatestCheckpoint();
    if (checkpoint.isPresent()) {
        LogPosition logPosition = checkpoint.get().getCheckpointEntryPosition();
        try (StoreChannel storeChannel = fileSystem.write(checkpointFile.getDetachedCheckpointFileForVersion(logPosition.getLogVersion()))) {
            storeChannel.truncate(logPosition.getByteOffset());
        }
    }
}

Example 3 with CheckpointFile

use of org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile in project neo4j by neo4j.

the class LogPruningTest method mustLogLatestPreservedCheckpointVersion.

@Test
void mustLogLatestPreservedCheckpointVersion() throws IOException {
    // given
    when(factory.strategyFromConfigValue(eq(fs), eq(logFiles), eq(logProvider), eq(clock), anyString())).thenReturn(x -> LongStream.empty());
    int checkpointLogFilesToKeep = config.get(checkpoint_logical_log_keep_threshold);
    CheckpointFile checkpointFile = mock(CheckpointFile.class);
    Path[] checkpointFiles = new Path[checkpointLogFilesToKeep + 2];
    for (int i = 0; i < checkpointFiles.length; i++) {
        checkpointFiles[i] = Paths.get(String.valueOf(i));
    }
    when(checkpointFile.getDetachedCheckpointFiles()).thenReturn(checkpointFiles);
    when(checkpointFile.getDetachedCheckpointLogFileVersion(any())).thenAnswer(invocationOnMock -> {
        Path file = invocationOnMock.getArgument(0);
        return Long.parseLong(file.getFileName().toString());
    });
    when(checkpointFile.getCurrentDetachedLogVersion()).thenReturn((long) checkpointFiles.length - 1);
    when(logFiles.getCheckpointFile()).thenReturn(checkpointFile);
    // when
    LogPruning pruning = new LogPruningImpl(fs, logFiles, logProvider, factory, clock, config);
    pruning.pruneLogs(1);
    // then
    verify(fs).deleteFile(checkpointFiles[0]);
    verify(fs).deleteFile(checkpointFiles[1]);
    LogAssertions.assertThat(logProvider).forLevel(INFO).forClass(LogPruningImpl.class).containsMessages("Pruned 2 checkpoint log files. Lowest preserved version: 2");
}

Example 4 with CheckpointFile

use of org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile in project neo4j by neo4j.

the class CorruptedLogsTruncatorTest method pruneAndArchiveMultipleLogs.

@Test
void pruneAndArchiveMultipleLogs() throws IOException {
    life.start();
    generateTransactionLogFiles(logFiles);
    long highestCorrectLogFileIndex = 5;
    var logFile = logFiles.getLogFile();
    Path highestCorrectLogFile = logFile.getLogFileForVersion(highestCorrectLogFileIndex);
    long fileSizeBeforePrune = Files.size(highestCorrectLogFile);
    long highestLogFileLength = Files.size(logFile.getHighestLogFile());
    int bytesToPrune = 7;
    long byteOffset = fileSizeBeforePrune - bytesToPrune;
    LogPosition prunePosition = new LogPosition(highestCorrectLogFileIndex, byteOffset);
    CheckpointFile checkpointFile = logFiles.getCheckpointFile();
    checkpointFile.getCheckpointAppender().checkPoint(LogCheckPointEvent.NULL, new LogPosition(highestCorrectLogFileIndex, byteOffset - 1), Instant.now(), "within okay transactions");
    /* Write checkpoints that should be truncated. Write enough to get them get them in two files. */
    for (int i = 0; i < 5; i++) {
        checkpointFile.getCheckpointAppender().checkPoint(LogCheckPointEvent.NULL, new LogPosition(highestCorrectLogFileIndex, byteOffset + 1), Instant.now(), "in the part being truncated");
    }
    life.shutdown();
    logPruner.truncate(prunePosition);
    life.start();
    // 6 transaction log files and a checkpoint file
    logVersionRepository.setCheckpointLogVersion(0, NULL);
    assertEquals(7, logFiles.logFiles().length);
    assertEquals(byteOffset, Files.size(highestCorrectLogFile));
    assertThat(checkpointFile.getDetachedCheckpointFiles()).hasSize(1);
    assertEquals(CURRENT_FORMAT_LOG_HEADER_SIZE + 192, /* one checkpoint */
    Files.size(checkpointFile.getDetachedCheckpointFiles()[0]));
    Path corruptedLogsDirectory = databaseDirectory.resolve(CORRUPTED_TX_LOGS_BASE_NAME);
    assertTrue(Files.exists(corruptedLogsDirectory));
    File[] files = corruptedLogsDirectory.toFile().listFiles();
    assertNotNull(files);
    assertEquals(1, files.length);
    File corruptedLogsArchive = files[0];
    checkArchiveName(highestCorrectLogFileIndex, byteOffset, corruptedLogsArchive);
    try (ZipFile zipFile = new ZipFile(corruptedLogsArchive)) {
        assertEquals(9, zipFile.size());
        checkEntryNameAndSize(zipFile, highestCorrectLogFile.getFileName().toString(), bytesToPrune);
        long nextLogFileIndex = highestCorrectLogFileIndex + 1;
        int lastFileIndex = TOTAL_NUMBER_OF_TRANSACTION_LOG_FILES - 1;
        for (long index = nextLogFileIndex; index < lastFileIndex; index++) {
            checkEntryNameAndSize(zipFile, TransactionLogFilesHelper.DEFAULT_NAME + "." + index, SINGLE_LOG_FILE_SIZE);
        }
        checkEntryNameAndSize(zipFile, TransactionLogFilesHelper.DEFAULT_NAME + "." + lastFileIndex, highestLogFileLength);
        checkEntryNameAndSize(zipFile, TransactionLogFilesHelper.CHECKPOINT_FILE_PREFIX + ".0", 192 * 4);
        checkEntryNameAndSize(zipFile, TransactionLogFilesHelper.CHECKPOINT_FILE_PREFIX + ".1", CURRENT_FORMAT_LOG_HEADER_SIZE + 192);
    }
}

Example 5 with CheckpointFile

use of org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile in project neo4j by neo4j.

the class CorruptedLogsTruncator method truncateLogFiles.

private void truncateLogFiles(long recoveredTransactionLogVersion, long recoveredTransactionOffset, Optional<CheckpointInfo> corruptCheckpoint) throws IOException {
    LogFile transactionLogFile = logFiles.getLogFile();
    truncateFilesFromVersion(recoveredTransactionLogVersion, recoveredTransactionOffset, transactionLogFile.getHighestLogVersion(), transactionLogFile::getLogFileForVersion);
    if (corruptCheckpoint.isPresent()) {
        LogPosition checkpointPosition = corruptCheckpoint.get().getCheckpointEntryPosition();
        CheckpointFile checkpointFile = logFiles.getCheckpointFile();
        truncateFilesFromVersion(checkpointPosition.getLogVersion(), checkpointPosition.getByteOffset(), checkpointFile.getCurrentDetachedLogVersion(), checkpointFile::getDetachedCheckpointFileForVersion);
    }
}