Examples with VersionAwareLogEntryReader org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader used on opensource projects

Example 16 with VersionAwareLogEntryReader

use of org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader in project neo4j by neo4j.

the class TransactionLogFileRotateAndReadRaceIT method shouldNotSeeEmptyLogFileWhenReadingTransactionStream.

@Test
void shouldNotSeeEmptyLogFileWhenReadingTransactionStream() throws Exception {
    // GIVEN
    LogVersionRepository logVersionRepository = new SimpleLogVersionRepository();
    Config cfg = Config.newBuilder().set(GraphDatabaseSettings.neo4j_home, databaseLayout.getNeo4jLayout().homeDirectory()).set(GraphDatabaseSettings.preallocate_logical_logs, false).set(GraphDatabaseSettings.logical_log_rotation_threshold, ByteUnit.kibiBytes(128)).build();
    LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogVersionRepository(logVersionRepository).withTransactionIdStore(new SimpleTransactionIdStore()).withLogEntryReader(new VersionAwareLogEntryReader(new TestCommandReaderFactory())).withConfig(cfg).withStoreId(StoreId.UNKNOWN).build();
    life.add(logFiles);
    LogFile logFile = logFiles.getLogFile();
    var writer = logFile.getTransactionLogWriter();
    LogPositionMarker startPosition = new LogPositionMarker();
    writer.getCurrentPosition(startPosition);
    // WHEN
    AtomicBoolean end = new AtomicBoolean();
    byte[] dataChunk = new byte[100];
    // one thread constantly writing to and rotating the channel
    CountDownLatch startSignal = new CountDownLatch(1);
    Future<Void> writeFuture = t2.execute(() -> {
        ThreadLocalRandom random = ThreadLocalRandom.current();
        startSignal.countDown();
        int rotations = 0;
        while (!end.get()) {
            int bytesToWrite = random.nextInt(1, dataChunk.length);
            writer.getChannel().put(dataChunk, bytesToWrite);
            if (logFile.rotationNeeded()) {
                logFile.rotate();
                // Let's just close the gap to the reader so that it gets closer to the "hot zone"
                // where the rotation happens.
                writer.getCurrentPosition(startPosition);
                if (++rotations > LIMIT_ROTATIONS) {
                    end.set(true);
                }
            }
        }
        return null;
    });
    assertTrue(startSignal.await(10, SECONDS));
    // one thread reading through the channel
    try {
        int reads = 0;
        while (!end.get()) {
            try (ReadableLogChannel reader = logFile.getReader(startPosition.newPosition())) {
                deplete(reader);
            }
            if (++reads > LIMIT_READS) {
                end.set(true);
            }
        }
    } finally {
        writeFuture.get();
    }
// THEN simply getting here means this was successful
}

Example 17 with VersionAwareLogEntryReader

use of org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader in project neo4j by neo4j.

the class RecoverIndexDropIT method appendDropTransactionToTransactionLog.

private void appendDropTransactionToTransactionLog(Path transactionLogsDirectory, CommittedTransactionRepresentation dropTransaction, StorageEngineFactory storageEngineFactory) throws IOException {
    LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder(transactionLogsDirectory, fs).withCommandReaderFactory(storageEngineFactory.commandReaderFactory()).build();
    LogFile logFile = logFiles.getLogFile();
    try (ReadableLogChannel reader = logFile.getReader(logFile.extractHeader(0).getStartPosition())) {
        LogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
        while (logEntryReader.readLogEntry(reader) != null) {
        }
        LogPosition position = logEntryReader.lastPosition();
        StoreChannel storeChannel = fs.write(logFile.getLogFileForVersion(logFile.getHighestLogVersion()));
        storeChannel.position(position.getByteOffset());
        try (PhysicalFlushableChecksumChannel writeChannel = new PhysicalFlushableChecksumChannel(storeChannel, new HeapScopedBuffer(100, INSTANCE))) {
            new LogEntryWriter<>(writeChannel, KernelVersion.LATEST).serialize(dropTransaction);
        }
    }
}

Example 18 with VersionAwareLogEntryReader

use of org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader in project neo4j by neo4j.

the class LogFilesBuilder method buildContext.

TransactionLogFilesContext buildContext() throws IOException {
    if (logEntryReader == null) {
        requireNonNull(commandReaderFactory);
        logEntryReader = new VersionAwareLogEntryReader(commandReaderFactory);
    }
    if (config == null) {
        config = Config.defaults();
    }
    requireNonNull(fileSystem);
    Supplier<StoreId> storeIdSupplier = getStoreId();
    Supplier<LogVersionRepository> logVersionRepositorySupplier = getLogVersionRepositorySupplier();
    LongSupplier lastCommittedIdSupplier = lastCommittedIdSupplier();
    LongSupplier committingTransactionIdSupplier = committingIdSupplier();
    Supplier<LogPosition> lastClosedTransactionPositionSupplier = closePositionSupplier();
    // Register listener for rotation threshold
    AtomicLong rotationThreshold = getRotationThresholdAndRegisterForUpdates();
    AtomicBoolean tryPreallocateTransactionLogs = getTryToPreallocateTransactionLogs();
    var nativeAccess = getNativeAccess();
    var monitors = getMonitors();
    var health = getDatabaseHealth();
    var clock = getClock();
    // Or the latest version if we can't find the system db version.
    if (kernelVersionRepository == null) {
        kernelVersionRepository = () -> KernelVersion.LATEST;
        if (dependencies != null) {
            try {
                this.kernelVersionRepository = dependencies.resolveDependency(KernelVersionRepository.class);
            } catch (UnsatisfiedDependencyException e) {
            // Use latest version if can't find version repository.
            }
        }
    }
    return new TransactionLogFilesContext(rotationThreshold, tryPreallocateTransactionLogs, logEntryReader, lastCommittedIdSupplier, committingTransactionIdSupplier, lastClosedTransactionPositionSupplier, logVersionRepositorySupplier, fileSystem, logProvider, databaseTracers, storeIdSupplier, nativeAccess, memoryTracker, monitors, config.get(fail_on_corrupted_log_files), health, kernelVersionRepository, clock, config);
}

Example 19 with VersionAwareLogEntryReader

use of org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader in project neo4j by neo4j.

the class TransactionLogInitializer method initializeExistingLogFiles.

/**
 * Make sure that any existing log files in the given transaction logs directory are initialised.
 * This is done when we migrate 3.x stores into a 4.x world.
 */
public void initializeExistingLogFiles(DatabaseLayout layout, Path transactionLogsDirectory, String checkpointReason) throws Exception {
    try (LogFilesSpan span = buildLogFiles(layout, transactionLogsDirectory)) {
        LogFiles logFiles = span.getLogFiles();
        LogFile logFile = logFiles.getLogFile();
        LogHeader logHeader = logFile.extractHeader(logFile.getLowestLogVersion());
        VersionAwareLogEntryReader entryReader = new VersionAwareLogEntryReader(commandReaderFactory, false);
        try (var readableChannel = logFile.getReader(logHeader.getStartPosition());
            var cursor = new LogEntryCursor(entryReader, readableChannel)) {
            while (cursor.next()) {
                LogEntry entry = cursor.get();
                if (entry.getType() == LogEntryTypeCodes.TX_COMMIT) {
                    // The log files already contain a transaction, so we can only append checkpoint for the end of the log files.
                    appendCheckpoint(logFiles, checkpointReason, logFile.getTransactionLogWriter().getCurrentPosition());
                    return;
                }
            }
        }
        appendEmptyTransactionAndCheckPoint(logFiles, checkpointReason);
    }
}

Example 20 with VersionAwareLogEntryReader

use of org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader in project neo4j by neo4j.

the class RecoveryRequiredChecker method isRecoveryRequiredAt.

public boolean isRecoveryRequiredAt(DatabaseLayout databaseLayout, MemoryTracker memoryTracker) throws IOException {
    LogEntryReader reader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
    LogFiles logFiles = buildLogFiles(databaseLayout, reader, memoryTracker);
    return isRecoveryRequiredAt(databaseLayout, logFiles);
}