Examples with LogPositionMarker org.neo4j.kernel.impl.transaction.log.LogPositionMarker used on opensource projects

Example 6 with LogPositionMarker

use of org.neo4j.kernel.impl.transaction.log.LogPositionMarker in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method shouldTruncateInvalidCheckpointAndAllCorruptTransactions.

@Test
void shouldTruncateInvalidCheckpointAndAllCorruptTransactions() throws IOException {
    Path file = logFiles.getLogFile().getLogFileForVersion(logVersion);
    LogPositionMarker marker = new LogPositionMarker();
    writeSomeData(file, pair -> {
        LogEntryWriter writer = pair.first();
        writer.writeStartEntry(1L, 1L, BASE_TX_CHECKSUM, ArrayUtils.EMPTY_BYTE_ARRAY);
        writer.writeCommitEntry(1L, 2L);
        Consumer<LogPositionMarker> other = pair.other();
        other.accept(marker);
        var checkpointFile = logFiles.getCheckpointFile();
        var checkpointAppender = checkpointFile.getCheckpointAppender();
        checkpointAppender.checkPoint(LogCheckPointEvent.NULL, marker.newPosition(), Instant.now(), "valid checkpoint");
        checkpointAppender.checkPoint(LogCheckPointEvent.NULL, new LogPosition(marker.getLogVersion() + 1, marker.getByteOffset()), Instant.now(), "invalid checkpoint");
        // incomplete tx
        writer.writeStartEntry(5L, 4L, 0, new byte[0]);
        return true;
    });
    assertTrue(recover(storeDir, logFiles));
    assertEquals(marker.getByteOffset(), Files.size(file));
    assertEquals(CURRENT_FORMAT_LOG_HEADER_SIZE + 192, /* one checkpoint */
    Files.size(logFiles.getCheckpointFile().getCurrentFile()));
}

Example 7 with LogPositionMarker

use of org.neo4j.kernel.impl.transaction.log.LogPositionMarker in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method doNotTruncateCheckpointsAfterLastTransaction.

@Test
void doNotTruncateCheckpointsAfterLastTransaction() throws IOException {
    Path file = logFiles.getLogFile().getLogFileForVersion(logVersion);
    LogPositionMarker marker = new LogPositionMarker();
    writeSomeData(file, pair -> {
        LogEntryWriter writer = pair.first();
        writer.writeStartEntry(1L, 1L, BASE_TX_CHECKSUM, ArrayUtils.EMPTY_BYTE_ARRAY);
        writer.writeCommitEntry(1L, 2L);
        Consumer<LogPositionMarker> other = pair.other();
        other.accept(marker);
        var checkpointFile = logFiles.getCheckpointFile();
        var checkpointAppender = checkpointFile.getCheckpointAppender();
        checkpointAppender.checkPoint(LogCheckPointEvent.NULL, marker.newPosition(), Instant.now(), "test");
        // write incomplete tx to trigger recovery
        writer.writeStartEntry(5L, 4L, 0, new byte[0]);
        return true;
    });
    assertTrue(recover(storeDir, logFiles));
    assertEquals(marker.getByteOffset(), Files.size(file));
    assertEquals(CURRENT_FORMAT_LOG_HEADER_SIZE + 192, /* one checkpoint */
    Files.size(logFiles.getCheckpointFile().getCurrentFile()));
}

Example 8 with LogPositionMarker

use of org.neo4j.kernel.impl.transaction.log.LogPositionMarker in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method shouldFailRecoveryWhenCanceled.

@Test
void shouldFailRecoveryWhenCanceled() throws Exception {
    Path file = logFiles.getLogFile().getLogFileForVersion(logVersion);
    final LogPositionMarker marker = new LogPositionMarker();
    final byte[] additionalHeaderData = new byte[0];
    final long transactionId = 4;
    final long commitTimestamp = 5;
    writeSomeData(file, pair -> {
        LogEntryWriter writer = pair.first();
        Consumer<LogPositionMarker> consumer = pair.other();
        // last committed tx
        writer.writeStartEntry(2L, 3L, BASE_TX_CHECKSUM, additionalHeaderData);
        writer.writeCommitEntry(transactionId, commitTimestamp);
        consumer.accept(marker);
        return true;
    });
    RecoveryMonitor monitor = mock(RecoveryMonitor.class);
    var startupController = mock(DatabaseStartupController.class);
    var databaseId = from("db", randomUUID());
    when(startupController.shouldAbort(databaseId)).thenReturn(false, true);
    var recoveryStartupChecker = new RecoveryStartupChecker(startupController, databaseId);
    var logsTruncator = mock(CorruptedLogsTruncator.class);
    var exception = assertThrows(Exception.class, () -> recover(storeDir, logFiles, recoveryStartupChecker));
    var rootCause = getRootCause(exception);
    assertThat(rootCause).isInstanceOf(DatabaseStartAbortedException.class);
    verify(logsTruncator, never()).truncate(any());
    verify(monitor, never()).recoveryCompleted(anyInt(), anyLong());
}

Example 9 with LogPositionMarker

use of org.neo4j.kernel.impl.transaction.log.LogPositionMarker in project neo4j by neo4j.

the class RecoveryTest method shouldRecoverExistingData.

@Test
public void shouldRecoverExistingData() throws Exception {
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(directory.directory(), "log", fileSystemRule.get());
    File file = logFiles.getLogFileForVersion(logVersion);
    writeSomeData(file, new Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException>() {

        @Override
        public boolean visit(Pair<LogEntryWriter, Consumer<LogPositionMarker>> pair) throws IOException {
            LogEntryWriter writer = pair.first();
            Consumer<LogPositionMarker> consumer = pair.other();
            LogPositionMarker marker = new LogPositionMarker();
            // last committed tx
            consumer.accept(marker);
            LogPosition lastCommittedTxPosition = marker.newPosition();
            writer.writeStartEntry(0, 1, 2L, 3L, new byte[0]);
            lastCommittedTxStartEntry = new LogEntryStart(0, 1, 2L, 3L, new byte[0], lastCommittedTxPosition);
            writer.writeCommitEntry(4L, 5L);
            lastCommittedTxCommitEntry = new OnePhaseCommit(4L, 5L);
            // check point pointing to the previously committed transaction
            writer.writeCheckPointEntry(lastCommittedTxPosition);
            expectedCheckPointEntry = new CheckPoint(lastCommittedTxPosition);
            // tx committed after checkpoint
            consumer.accept(marker);
            writer.writeStartEntry(0, 1, 6L, 4L, new byte[0]);
            expectedStartEntry = new LogEntryStart(0, 1, 6L, 4L, new byte[0], marker.newPosition());
            writer.writeCommitEntry(5L, 7L);
            expectedCommitEntry = new OnePhaseCommit(5L, 7L);
            return true;
        }
    });
    LifeSupport life = new LifeSupport();
    Recovery.Monitor monitor = mock(Recovery.Monitor.class);
    final AtomicBoolean recoveryRequired = new AtomicBoolean();
    try {
        StorageEngine storageEngine = mock(StorageEngine.class);
        final LogEntryReader<ReadableClosablePositionAwareChannel> reader = new VersionAwareLogEntryReader<>();
        LatestCheckPointFinder finder = new LatestCheckPointFinder(logFiles, fileSystemRule.get(), reader);
        LogHeaderCache logHeaderCache = new LogHeaderCache(10);
        TransactionMetadataCache metadataCache = new TransactionMetadataCache(100);
        LogFile logFile = life.add(new PhysicalLogFile(fileSystemRule.get(), logFiles, 50, () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, mock(PhysicalLogFile.Monitor.class), logHeaderCache));
        LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore(logFile, metadataCache, reader);
        life.add(new Recovery(new DefaultRecoverySPI(storageEngine, logFiles, fileSystemRule.get(), logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR) {

            private int nr = 0;

            @Override
            public Visitor<CommittedTransactionRepresentation, Exception> startRecovery() {
                recoveryRequired.set(true);
                final Visitor<CommittedTransactionRepresentation, Exception> actual = super.startRecovery();
                return new Visitor<CommittedTransactionRepresentation, Exception>() {

                    @Override
                    public boolean visit(CommittedTransactionRepresentation tx) throws Exception {
                        actual.visit(tx);
                        switch(nr++) {
                            case 0:
                                assertEquals(lastCommittedTxStartEntry, tx.getStartEntry());
                                assertEquals(lastCommittedTxCommitEntry, tx.getCommitEntry());
                                break;
                            case 1:
                                assertEquals(expectedStartEntry, tx.getStartEntry());
                                assertEquals(expectedCommitEntry, tx.getCommitEntry());
                                break;
                            default:
                                fail("Too many recovered transactions");
                        }
                        return false;
                    }
                };
            }
        }, monitor));
        life.start();
        InOrder order = inOrder(monitor);
        order.verify(monitor, times(1)).recoveryRequired(any(LogPosition.class));
        order.verify(monitor, times(1)).recoveryCompleted(2);
        assertTrue(recoveryRequired.get());
    } finally {
        life.shutdown();
    }
}

Example 10 with LogPositionMarker

use of org.neo4j.kernel.impl.transaction.log.LogPositionMarker in project neo4j by neo4j.

the class RecoveryTest method shouldTellTransactionIdStoreAfterSuccessfullRecovery.

@Test
public void shouldTellTransactionIdStoreAfterSuccessfullRecovery() throws Exception {
    // GIVEN
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(directory.directory(), "log", fileSystemRule.get());
    File file = logFiles.getLogFileForVersion(logVersion);
    final LogPositionMarker marker = new LogPositionMarker();
    final byte[] additionalHeaderData = new byte[0];
    final int masterId = 0;
    final int authorId = 1;
    final long transactionId = 4;
    final long commitTimestamp = 5;
    writeSomeData(file, new Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException>() {

        @Override
        public boolean visit(Pair<LogEntryWriter, Consumer<LogPositionMarker>> pair) throws IOException {
            LogEntryWriter writer = pair.first();
            Consumer<LogPositionMarker> consumer = pair.other();
            // last committed tx
            writer.writeStartEntry(masterId, authorId, 2L, 3L, additionalHeaderData);
            writer.writeCommitEntry(transactionId, commitTimestamp);
            consumer.accept(marker);
            return true;
        }
    });
    // WHEN
    boolean recoveryRequired = recover(logFiles);
    // THEN
    assertTrue(recoveryRequired);
    long[] lastClosedTransaction = transactionIdStore.getLastClosedTransaction();
    assertEquals(transactionId, lastClosedTransaction[0]);
    assertEquals(LogEntryStart.checksum(additionalHeaderData, masterId, authorId), transactionIdStore.getLastCommittedTransaction().checksum());
    assertEquals(commitTimestamp, transactionIdStore.getLastCommittedTransaction().commitTimestamp());
    assertEquals(logVersion, lastClosedTransaction[1]);
    assertEquals(marker.getByteOffset(), lastClosedTransaction[2]);
}