Examples with LogEntryWriter org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter used on opensource projects

Example 6 with LogEntryWriter

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter in project neo4j by neo4j.

the class StoreCopyClient method writeTransactionsToActiveLogFile.

private void writeTransactionsToActiveLogFile(File tempStoreDir, Response<?> response) throws Exception {
    LifeSupport life = new LifeSupport();
    try {
        // Start the log and appender
        PhysicalLogFiles logFiles = new PhysicalLogFiles(tempStoreDir, fs);
        LogHeaderCache logHeaderCache = new LogHeaderCache(10);
        ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository(pageCache, tempStoreDir);
        ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore(pageCache, tempStoreDir);
        LogFile logFile = life.add(new PhysicalLogFile(fs, logFiles, Long.MAX_VALUE, /*don't rotate*/
        readOnlyTransactionIdStore::getLastCommittedTransactionId, logVersionRepository, new Monitors().newMonitor(PhysicalLogFile.Monitor.class), logHeaderCache));
        life.start();
        // Just write all transactions to the active log version. Remember that this is after a store copy
        // where there are no logs, and the transaction stream we're about to write will probably contain
        // transactions that goes some time back, before the last committed transaction id. So we cannot
        // use a TransactionAppender, since it has checks for which transactions one can append.
        FlushableChannel channel = logFile.getWriter();
        final TransactionLogWriter writer = new TransactionLogWriter(new LogEntryWriter(channel));
        final AtomicLong firstTxId = new AtomicLong(BASE_TX_ID);
        response.accept(new Response.Handler() {

            @Override
            public void obligation(long txId) throws IOException {
                throw new UnsupportedOperationException("Shouldn't be called");
            }

            @Override
            public Visitor<CommittedTransactionRepresentation, Exception> transactions() {
                return transaction -> {
                    long txId = transaction.getCommitEntry().getTxId();
                    if (firstTxId.compareAndSet(BASE_TX_ID, txId)) {
                        monitor.startReceivingTransactions(txId);
                    }
                    writer.append(transaction.getTransactionRepresentation(), txId);
                    return false;
                };
            }
        });
        long endTxId = firstTxId.get();
        if (endTxId != BASE_TX_ID) {
            monitor.finishReceivingTransactions(endTxId);
        }
        long currentLogVersion = logVersionRepository.getCurrentLogVersion();
        writer.checkPoint(new LogPosition(currentLogVersion, LOG_HEADER_SIZE));
        // And since we write this manually we need to set the correct transaction id in the
        // header of the log that we just wrote.
        File currentLogFile = logFiles.getLogFileForVersion(currentLogVersion);
        writeLogHeader(fs, currentLogFile, currentLogVersion, max(BASE_TX_ID, endTxId - 1));
        if (!forensics) {
            // since we just create new log and put checkpoint into it with offset equals to
            // LOG_HEADER_SIZE we need to update last transaction offset to be equal to this newly defined max
            // offset otherwise next checkpoint that use last transaction offset will be created for non
            // existing offset that is in most of the cases bigger than new log size.
            // Recovery will treat that as last checkpoint and will not try to recover store till new
            // last closed transaction offset will not overcome old one. Till that happens it will be
            // impossible for recovery process to restore the store
            File neoStore = new File(tempStoreDir, MetaDataStore.DEFAULT_NAME);
            MetaDataStore.setRecord(pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, LOG_HEADER_SIZE);
        }
    } finally {
        life.shutdown();
    }
}

Example 7 with LogEntryWriter

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter in project neo4j by neo4j.

the class CheckTxLogsTest method writeContent.

private void writeContent(File log, ThrowingConsumer<TransactionLogWriter, IOException> consumer) throws IOException {
    FileSystemAbstraction fs = ensureLogExists(log);
    try (StoreChannel channel = fs.open(log, "rw");
        LogVersionedStoreChannel versionedChannel = new PhysicalLogVersionedStoreChannel(channel, 0, (byte) 0);
        PhysicalFlushableChannel writableLogChannel = new PhysicalFlushableChannel(versionedChannel)) {
        long offset = channel.size();
        channel.position(offset);
        consumer.accept(new TransactionLogWriter(new LogEntryWriter(writableLogChannel)));
    }
}

Example 8 with LogEntryWriter

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter in project neo4j by neo4j.

the class CoreBootstrapper method appendNullTransactionLogEntryToSetRaftIndexToMinusOne.

private void appendNullTransactionLogEntryToSetRaftIndexToMinusOne() throws IOException {
    PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, fs);
    ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository(pageCache, storeDir);
    ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore(pageCache, storeDir);
    PhysicalLogFile logFile = new PhysicalLogFile(fs, logFiles, Long.MAX_VALUE, /*don't rotate*/
    () -> readOnlyTransactionIdStore.getLastClosedTransactionId() - 1, logVersionRepository, new Monitors().newMonitor(PhysicalLogFile.Monitor.class), new LogHeaderCache(10));
    long dummyTransactionId;
    try (Lifespan lifespan = new Lifespan(logFile)) {
        FlushableChannel channel = logFile.getWriter();
        TransactionLogWriter writer = new TransactionLogWriter(new LogEntryWriter(channel));
        long lastCommittedTransactionId = readOnlyTransactionIdStore.getLastCommittedTransactionId();
        PhysicalTransactionRepresentation tx = new PhysicalTransactionRepresentation(Collections.emptyList());
        byte[] txHeaderBytes = LogIndexTxHeaderEncoding.encodeLogIndexAsTxHeader(-1);
        tx.setHeader(txHeaderBytes, -1, -1, -1, lastCommittedTransactionId, -1, -1);
        dummyTransactionId = lastCommittedTransactionId + 1;
        writer.append(tx, dummyTransactionId);
        channel.prepareForFlush().flush();
    }
    File neoStoreFile = new File(storeDir, MetaDataStore.DEFAULT_NAME);
    MetaDataStore.setRecord(pageCache, neoStoreFile, LAST_TRANSACTION_ID, dummyTransactionId);
}

Example 9 with LogEntryWriter

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter in project neo4j by neo4j.

the class LegacyLogEntryWriter method writeAllLogEntries.

public void writeAllLogEntries(LogVersionedStoreChannel channel, IOCursor<LogEntry> cursor) throws IOException {
    try (PositionAwarePhysicalFlushableChannel writable = new PositionAwarePhysicalFlushableChannel(channel)) {
        final LogEntryWriter writer = factory.apply(writable);
        List<StorageCommand> commands = new ArrayList<>();
        while (cursor.next()) {
            LogEntry entry = cursor.get();
            if (entry instanceof LogEntryStart) {
                final LogEntryStart startEntry = entry.as();
                writer.writeStartEntry(startEntry.getMasterId(), startEntry.getLocalId(), startEntry.getTimeWritten(), startEntry.getLastCommittedTxWhenTransactionStarted(), startEntry.getAdditionalHeader());
            } else if (entry instanceof LogEntryCommit) {
                if (!commands.isEmpty()) {
                    writer.serialize(new PhysicalTransactionRepresentation(commands));
                    commands = new ArrayList<>();
                }
                final LogEntryCommit commitEntry = (LogEntryCommit) entry;
                writer.writeCommitEntry(commitEntry.getTxId(), commitEntry.getTimeWritten());
            } else if (entry instanceof LogEntryCommand) {
                commands.add(((LogEntryCommand) entry).getXaCommand());
            } else {
                throw new IllegalStateException("Unknown entry: " + entry);
            }
        }
    }
}

Example 10 with LogEntryWriter

use of org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter in project neo4j by neo4j.

the class RecoveryTest method shouldRecoverExistingData.

@Test
public void shouldRecoverExistingData() throws Exception {
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(directory.directory(), "log", fileSystemRule.get());
    File file = logFiles.getLogFileForVersion(logVersion);
    writeSomeData(file, new Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException>() {

        @Override
        public boolean visit(Pair<LogEntryWriter, Consumer<LogPositionMarker>> pair) throws IOException {
            LogEntryWriter writer = pair.first();
            Consumer<LogPositionMarker> consumer = pair.other();
            LogPositionMarker marker = new LogPositionMarker();
            // last committed tx
            consumer.accept(marker);
            LogPosition lastCommittedTxPosition = marker.newPosition();
            writer.writeStartEntry(0, 1, 2L, 3L, new byte[0]);
            lastCommittedTxStartEntry = new LogEntryStart(0, 1, 2L, 3L, new byte[0], lastCommittedTxPosition);
            writer.writeCommitEntry(4L, 5L);
            lastCommittedTxCommitEntry = new OnePhaseCommit(4L, 5L);
            // check point pointing to the previously committed transaction
            writer.writeCheckPointEntry(lastCommittedTxPosition);
            expectedCheckPointEntry = new CheckPoint(lastCommittedTxPosition);
            // tx committed after checkpoint
            consumer.accept(marker);
            writer.writeStartEntry(0, 1, 6L, 4L, new byte[0]);
            expectedStartEntry = new LogEntryStart(0, 1, 6L, 4L, new byte[0], marker.newPosition());
            writer.writeCommitEntry(5L, 7L);
            expectedCommitEntry = new OnePhaseCommit(5L, 7L);
            return true;
        }
    });
    LifeSupport life = new LifeSupport();
    Recovery.Monitor monitor = mock(Recovery.Monitor.class);
    final AtomicBoolean recoveryRequired = new AtomicBoolean();
    try {
        StorageEngine storageEngine = mock(StorageEngine.class);
        final LogEntryReader<ReadableClosablePositionAwareChannel> reader = new VersionAwareLogEntryReader<>();
        LatestCheckPointFinder finder = new LatestCheckPointFinder(logFiles, fileSystemRule.get(), reader);
        LogHeaderCache logHeaderCache = new LogHeaderCache(10);
        TransactionMetadataCache metadataCache = new TransactionMetadataCache(100);
        LogFile logFile = life.add(new PhysicalLogFile(fileSystemRule.get(), logFiles, 50, () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, mock(PhysicalLogFile.Monitor.class), logHeaderCache));
        LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore(logFile, metadataCache, reader);
        life.add(new Recovery(new DefaultRecoverySPI(storageEngine, logFiles, fileSystemRule.get(), logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR) {

            private int nr = 0;

            @Override
            public Visitor<CommittedTransactionRepresentation, Exception> startRecovery() {
                recoveryRequired.set(true);
                final Visitor<CommittedTransactionRepresentation, Exception> actual = super.startRecovery();
                return new Visitor<CommittedTransactionRepresentation, Exception>() {

                    @Override
                    public boolean visit(CommittedTransactionRepresentation tx) throws Exception {
                        actual.visit(tx);
                        switch(nr++) {
                            case 0:
                                assertEquals(lastCommittedTxStartEntry, tx.getStartEntry());
                                assertEquals(lastCommittedTxCommitEntry, tx.getCommitEntry());
                                break;
                            case 1:
                                assertEquals(expectedStartEntry, tx.getStartEntry());
                                assertEquals(expectedCommitEntry, tx.getCommitEntry());
                                break;
                            default:
                                fail("Too many recovered transactions");
                        }
                        return false;
                    }
                };
            }
        }, monitor));
        life.start();
        InOrder order = inOrder(monitor);
        order.verify(monitor, times(1)).recoveryRequired(any(LogPosition.class));
        order.verify(monitor, times(1)).recoveryCompleted(2);
        assertTrue(recoveryRequired.get());
    } finally {
        life.shutdown();
    }
}