Examples with LogEntryStart org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart used on opensource projects

Example 16 with LogEntryStart

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart in project neo4j by neo4j.

the class LegacyLogEntryWriterTest method shouldWriteAllTheEntryInSeveralCommitsToTheFile.

@Test
public void shouldWriteAllTheEntryInSeveralCommitsToTheFile() throws IOException {
    // given
    final LogVersionedStoreChannel channel = mock(LogVersionedStoreChannel.class);
    final LogEntryWriter logEntryWriter = mock(LogEntryWriter.class);
    final LegacyLogEntryWriter writer = new LegacyLogEntryWriter(fs, liftToFactory(logEntryWriter));
    final LogEntryStart start1 = new LogEntryStart(0, 1, 2L, 3L, EMPTY_ADDITIONAL_ARRAY, UNSPECIFIED);
    final LogEntryCommand command1 = new LogEntryCommand(new Command.NodeCommand(nodeRecord, nodeRecord));
    final LogEntryCommit commit1 = new OnePhaseCommit(42L, 43L);
    final LogEntryStart start2 = new LogEntryStart(9, 8, 7L, 6L, EMPTY_ADDITIONAL_ARRAY, UNSPECIFIED);
    final LogEntryCommand command2 = new LogEntryCommand(new Command.RelationshipCommand(relRecord, relRecord));
    final LogEntryCommit commit2 = new OnePhaseCommit(84L, 85L);
    // when
    IOCursor<LogEntry> cursor = mockCursor(start1, command1, commit1, start2, command2, commit2);
    writer.writeAllLogEntries(channel, cursor);
    // then
    verify(logEntryWriter, times(1)).writeStartEntry(0, 1, 2L, 3L, EMPTY_ADDITIONAL_ARRAY);
    final TransactionRepresentation expected1 = new PhysicalTransactionRepresentation(Arrays.asList(command1.getXaCommand()));
    verify(logEntryWriter, times(1)).serialize(eq(expected1));
    verify(logEntryWriter, times(1)).writeCommitEntry(42L, 43L);
    verify(logEntryWriter, times(1)).writeStartEntry(9, 8, 7L, 6L, EMPTY_ADDITIONAL_ARRAY);
    final TransactionRepresentation expected2 = new PhysicalTransactionRepresentation(Arrays.asList(command2.getXaCommand()));
    verify(logEntryWriter, times(1)).serialize(eq(expected2));
    verify(logEntryWriter, times(1)).writeCommitEntry(84L, 85L);
}

Example 17 with LogEntryStart

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method shouldRecoverExistingData.

@Test
void shouldRecoverExistingData() throws Exception {
    LogFile logFile = logFiles.getLogFile();
    Path file = logFile.getLogFileForVersion(logVersion);
    writeSomeData(file, pair -> {
        LogEntryWriter writer = pair.first();
        Consumer<LogPositionMarker> consumer = pair.other();
        LogPositionMarker marker = new LogPositionMarker();
        // last committed tx
        int previousChecksum = BASE_TX_CHECKSUM;
        consumer.accept(marker);
        LogPosition lastCommittedTxPosition = marker.newPosition();
        writer.writeStartEntry(2L, 3L, previousChecksum, new byte[0]);
        lastCommittedTxStartEntry = new LogEntryStart(2L, 3L, previousChecksum, new byte[0], lastCommittedTxPosition);
        previousChecksum = writer.writeCommitEntry(4L, 5L);
        lastCommittedTxCommitEntry = new LogEntryCommit(4L, 5L, previousChecksum);
        // check point pointing to the previously committed transaction
        var checkpointFile = logFiles.getCheckpointFile();
        var checkpointAppender = checkpointFile.getCheckpointAppender();
        checkpointAppender.checkPoint(LogCheckPointEvent.NULL, lastCommittedTxPosition, Instant.now(), "test");
        // tx committed after checkpoint
        consumer.accept(marker);
        writer.writeStartEntry(6L, 4L, previousChecksum, new byte[0]);
        expectedStartEntry = new LogEntryStart(6L, 4L, previousChecksum, new byte[0], marker.newPosition());
        previousChecksum = writer.writeCommitEntry(5L, 7L);
        expectedCommitEntry = new LogEntryCommit(5L, 7L, previousChecksum);
        return true;
    });
    LifeSupport life = new LifeSupport();
    var recoveryRequired = new AtomicBoolean();
    var recoveredTransactions = new MutableInt();
    RecoveryMonitor monitor = new RecoveryMonitor() {

        @Override
        public void recoveryRequired(LogPosition recoveryPosition) {
            recoveryRequired.set(true);
        }

        @Override
        public void recoveryCompleted(int numberOfRecoveredTransactions, long recoveryTimeInMilliseconds) {
            recoveredTransactions.setValue(numberOfRecoveredTransactions);
        }
    };
    try {
        StorageEngine storageEngine = mock(StorageEngine.class);
        final LogEntryReader reader = logEntryReader();
        TransactionMetadataCache metadataCache = new TransactionMetadataCache();
        LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore(logFiles, metadataCache, reader, monitors, false);
        CorruptedLogsTruncator logPruner = new CorruptedLogsTruncator(storeDir, logFiles, fileSystem, INSTANCE);
        monitors.addMonitorListener(monitor);
        life.add(new TransactionLogsRecovery(new DefaultRecoveryService(storageEngine, transactionIdStore, txStore, versionRepository, logFiles, NO_MONITOR, mock(Log.class), false) {

            private int nr;

            @Override
            public RecoveryApplier getRecoveryApplier(TransactionApplicationMode mode, PageCacheTracer cacheTracer, String tracerTag) {
                RecoveryApplier actual = super.getRecoveryApplier(mode, cacheTracer, tracerTag);
                if (mode == TransactionApplicationMode.REVERSE_RECOVERY) {
                    return actual;
                }
                return new RecoveryApplier() {

                    @Override
                    public void close() throws Exception {
                        actual.close();
                    }

                    @Override
                    public boolean visit(CommittedTransactionRepresentation tx) throws Exception {
                        actual.visit(tx);
                        switch(nr++) {
                            case 0:
                                assertEquals(lastCommittedTxStartEntry, tx.getStartEntry());
                                assertEquals(lastCommittedTxCommitEntry, tx.getCommitEntry());
                                break;
                            case 1:
                                assertEquals(expectedStartEntry, tx.getStartEntry());
                                assertEquals(expectedCommitEntry, tx.getCommitEntry());
                                break;
                            default:
                                fail("Too many recovered transactions");
                        }
                        return false;
                    }
                };
            }
        }, logPruner, schemaLife, monitor, ProgressReporter.SILENT, false, EMPTY_CHECKER, NULL));
        life.start();
        assertTrue(recoveryRequired.get());
        assertEquals(2, recoveredTransactions.getValue());
    } finally {
        life.shutdown();
    }
}

Example 18 with LogEntryStart

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart in project neo4j by neo4j.

the class DetachedLogTailScanner method getFirstTransactionIdAfterCheckpoint.

private StartCommitEntries getFirstTransactionIdAfterCheckpoint(LogFile logFile, LogPosition logPosition) throws IOException {
    boolean corruptedTransactionLogs = false;
    LogEntryStart start = null;
    LogEntryCommit commit = null;
    LogPosition lookupPosition = null;
    long logVersion = logPosition.getLogVersion();
    try {
        while (logFile.versionExists(logVersion)) {
            lookupPosition = lookupPosition == null ? logPosition : logFile.extractHeader(logVersion).getStartPosition();
            try (var reader = logFile.getReader(lookupPosition, NO_MORE_CHANNELS);
                var cursor = new LogEntryCursor(logEntryReader, reader)) {
                LogEntry entry;
                while ((start == null || commit == null) && cursor.next()) {
                    entry = cursor.get();
                    if (commit == null && entry instanceof LogEntryCommit) {
                        commit = (LogEntryCommit) entry;
                    } else if (start == null && entry instanceof LogEntryStart) {
                        start = (LogEntryStart) entry;
                    }
                }
            }
            if ((start != null) && (commit != null)) {
                return new StartCommitEntries(start, commit);
            }
            verifyReaderPosition(logVersion, logEntryReader.lastPosition());
            logVersion++;
        }
    } catch (Error | ClosedByInterruptException e) {
        // These should not be parsing errors
        throw e;
    } catch (Throwable t) {
        monitor.corruptedLogFile(logVersion, t);
        if (failOnCorruptedLogFiles) {
            throwUnableToCleanRecover(t);
        }
        corruptedTransactionLogs = true;
    }
    return new StartCommitEntries(start, commit, corruptedTransactionLogs);
}

Example 19 with LogEntryStart

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart in project neo4j by neo4j.

the class InlinedLogTailScanner method findLogTail.

protected LogTailInformation findLogTail() throws IOException {
    LogFile logFile = logFiles.getLogFile();
    final long highestLogVersion = logFile.getHighestLogVersion();
    long version = highestLogVersion;
    long versionToSearchForCommits = highestLogVersion;
    LogEntryStart latestStartEntry = null;
    long oldestStartEntryTransaction = NO_TRANSACTION_ID;
    long oldestVersionFound = -1;
    byte latestLogEntryVersion = 0;
    boolean startRecordAfterCheckpoint = false;
    boolean corruptedTransactionLogs = false;
    while (version >= logFile.getLowestLogVersion() && version >= INITIAL_LOG_VERSION) {
        log.info("Scanning log file with version %d for checkpoint entries", version);
        oldestVersionFound = version;
        CheckpointInfo latestCheckPoint = null;
        StoreId storeId = StoreId.UNKNOWN;
        try (LogVersionedStoreChannel channel = logFile.openForVersion(version);
            var readAheadChannel = new ReadAheadLogChannel(channel, memoryTracker);
            LogEntryCursor cursor = new LogEntryCursor(logEntryReader, readAheadChannel)) {
            LogHeader logHeader = logFile.extractHeader(version);
            storeId = logHeader.getStoreId();
            LogEntry entry;
            long position = logHeader.getStartPosition().getByteOffset();
            long channelVersion = version;
            while (cursor.next()) {
                entry = cursor.get();
                // Collect data about latest checkpoint
                if (entry instanceof LogEntryInlinedCheckPoint) {
                    latestCheckPoint = new CheckpointInfo((LogEntryInlinedCheckPoint) entry, storeId, new LogPosition(channelVersion, position));
                } else if (entry instanceof LogEntryCommit) {
                    if (oldestStartEntryTransaction == NO_TRANSACTION_ID) {
                        oldestStartEntryTransaction = ((LogEntryCommit) entry).getTxId();
                    }
                } else if (entry instanceof LogEntryStart) {
                    LogEntryStart startEntry = (LogEntryStart) entry;
                    if (version == versionToSearchForCommits) {
                        latestStartEntry = startEntry;
                    }
                    startRecordAfterCheckpoint = true;
                }
                // Collect data about latest entry version, only in first log file
                if (version == versionToSearchForCommits || latestLogEntryVersion == 0) {
                    latestLogEntryVersion = entry.getVersion().version();
                }
                position = channel.position();
                channelVersion = channel.getVersion();
            }
            verifyReaderPosition(version, logEntryReader.lastPosition());
        } catch (Error | ClosedByInterruptException e) {
            // These should not be parsing errors
            throw e;
        } catch (Throwable t) {
            monitor.corruptedLogFile(version, t);
            if (failOnCorruptedLogFiles) {
                throwUnableToCleanRecover(t);
            }
            corruptedTransactionLogs = true;
        }
        if (latestCheckPoint != null) {
            return checkpointTailInformation(highestLogVersion, latestStartEntry, oldestVersionFound, latestLogEntryVersion, latestCheckPoint, corruptedTransactionLogs, storeId);
        }
        version--;
        // if we have found no commits in the latest log, keep searching in the next one
        if (latestStartEntry == null) {
            versionToSearchForCommits--;
        }
    }
    return new LogTailInformation(corruptedTransactionLogs || startRecordAfterCheckpoint, oldestStartEntryTransaction, oldestVersionFound == UNKNOWN, highestLogVersion, latestLogEntryVersion);
}

Example 20 with LogEntryStart

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart in project neo4j by neo4j.

the class TransactionLogAppendAndRotateIT method assertWholeTransactionsIn.

private static void assertWholeTransactionsIn(LogFile logFile, long logVersion) throws IOException {
    try (ReadableLogChannel reader = logFile.getReader(new LogPosition(logVersion, CURRENT_FORMAT_LOG_HEADER_SIZE))) {
        LogEntryReader entryReader = logEntryReader();
        LogEntry entry;
        boolean inTx = false;
        int transactions = 0;
        while ((entry = entryReader.readLogEntry(reader)) != null) {
            if (// Expects start entry
            !inTx) {
                assertTrue(entry instanceof LogEntryStart);
                inTx = true;
            } else // Expects command/commit entry
            {
                assertTrue(entry instanceof LogEntryCommand || entry instanceof LogEntryCommit);
                if (entry instanceof LogEntryCommit) {
                    inTx = false;
                    transactions++;
                }
            }
        }
        assertFalse(inTx);
        assertTrue(transactions > 0);
    }
}