Examples with LogFile org.neo4j.kernel.impl.transaction.log.files.LogFile used on opensource projects

Example 6 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class TransactionLogFileTest method shouldWaitForOngoingForceToCompleteBeforeForcingAgain.

@Test
void shouldWaitForOngoingForceToCompleteBeforeForcingAgain() throws Throwable {
    LogFiles logFiles = buildLogFiles();
    life.start();
    life.add(logFiles);
    LogFile logFile = logFiles.getLogFile();
    var capturingChannel = wrappingFileSystem.getCapturingChannel();
    ReentrantLock writeAllLock = capturingChannel.getWriteAllLock();
    var flushesBefore = capturingChannel.getFlushCounter().get();
    var writesBefore = capturingChannel.getWriteAllCounter().get();
    writeAllLock.lock();
    int executors = 10;
    var executorService = Executors.newFixedThreadPool(executors);
    try {
        var future = executorService.submit(() -> logFile.forceAfterAppend(LogAppendEvent.NULL));
        while (!writeAllLock.hasQueuedThreads()) {
            parkNanos(100);
        }
        writeAllLock.unlock();
        var future2 = executorService.submit(() -> logFile.forceAfterAppend(LogAppendEvent.NULL));
        Futures.getAll(List.of(future, future2));
    } finally {
        if (writeAllLock.isLocked()) {
            writeAllLock.unlock();
        }
        executorService.shutdownNow();
    }
    assertThat(capturingChannel.getWriteAllCounter().get() - writesBefore).isEqualTo(2);
    assertThat(capturingChannel.getFlushCounter().get() - flushesBefore).isEqualTo(2);
}

Example 7 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class TransactionLogFileTest method closeChannelThrowExceptionOnAttemptToAppendTransactionLogRecords.

@Test
void closeChannelThrowExceptionOnAttemptToAppendTransactionLogRecords() throws IOException {
    LogFiles logFiles = buildLogFiles();
    life.start();
    life.add(logFiles);
    LogFile logFile = logFiles.getLogFile();
    var channel = logFile.getTransactionLogWriter().getChannel();
    life.shutdown();
    assertThrows(Throwable.class, () -> channel.put((byte) 7));
    assertThrows(Throwable.class, () -> channel.putInt(7));
    assertThrows(Throwable.class, () -> channel.putLong(7));
    assertThrows(Throwable.class, () -> channel.putDouble(7));
    assertThrows(Throwable.class, () -> channel.putFloat(7));
    assertThrows(Throwable.class, () -> channel.putShort((short) 7));
    assertThrows(Throwable.class, () -> channel.put(new byte[] { 1, 2, 3 }, 3));
    assertThrows(IllegalStateException.class, logFile::flush);
}

Example 8 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class TransactionLogFileRotateAndReadRaceIT method shouldNotSeeEmptyLogFileWhenReadingTransactionStream.

@Test
void shouldNotSeeEmptyLogFileWhenReadingTransactionStream() throws Exception {
    // GIVEN
    LogVersionRepository logVersionRepository = new SimpleLogVersionRepository();
    Config cfg = Config.newBuilder().set(GraphDatabaseSettings.neo4j_home, databaseLayout.getNeo4jLayout().homeDirectory()).set(GraphDatabaseSettings.preallocate_logical_logs, false).set(GraphDatabaseSettings.logical_log_rotation_threshold, ByteUnit.kibiBytes(128)).build();
    LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogVersionRepository(logVersionRepository).withTransactionIdStore(new SimpleTransactionIdStore()).withLogEntryReader(new VersionAwareLogEntryReader(new TestCommandReaderFactory())).withConfig(cfg).withStoreId(StoreId.UNKNOWN).build();
    life.add(logFiles);
    LogFile logFile = logFiles.getLogFile();
    var writer = logFile.getTransactionLogWriter();
    LogPositionMarker startPosition = new LogPositionMarker();
    writer.getCurrentPosition(startPosition);
    // WHEN
    AtomicBoolean end = new AtomicBoolean();
    byte[] dataChunk = new byte[100];
    // one thread constantly writing to and rotating the channel
    CountDownLatch startSignal = new CountDownLatch(1);
    Future<Void> writeFuture = t2.execute(() -> {
        ThreadLocalRandom random = ThreadLocalRandom.current();
        startSignal.countDown();
        int rotations = 0;
        while (!end.get()) {
            int bytesToWrite = random.nextInt(1, dataChunk.length);
            writer.getChannel().put(dataChunk, bytesToWrite);
            if (logFile.rotationNeeded()) {
                logFile.rotate();
                // Let's just close the gap to the reader so that it gets closer to the "hot zone"
                // where the rotation happens.
                writer.getCurrentPosition(startPosition);
                if (++rotations > LIMIT_ROTATIONS) {
                    end.set(true);
                }
            }
        }
        return null;
    });
    assertTrue(startSignal.await(10, SECONDS));
    // one thread reading through the channel
    try {
        int reads = 0;
        while (!end.get()) {
            try (ReadableLogChannel reader = logFile.getReader(startPosition.newPosition())) {
                deplete(reader);
            }
            if (++reads > LIMIT_READS) {
                end.set(true);
            }
        }
    } finally {
        writeFuture.get();
    }
// THEN simply getting here means this was successful
}

Example 9 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class RecoverIndexDropIT method appendDropTransactionToTransactionLog.

private void appendDropTransactionToTransactionLog(Path transactionLogsDirectory, CommittedTransactionRepresentation dropTransaction, StorageEngineFactory storageEngineFactory) throws IOException {
    LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder(transactionLogsDirectory, fs).withCommandReaderFactory(storageEngineFactory.commandReaderFactory()).build();
    LogFile logFile = logFiles.getLogFile();
    try (ReadableLogChannel reader = logFile.getReader(logFile.extractHeader(0).getStartPosition())) {
        LogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
        while (logEntryReader.readLogEntry(reader) != null) {
        }
        LogPosition position = logEntryReader.lastPosition();
        StoreChannel storeChannel = fs.write(logFile.getLogFileForVersion(logFile.getHighestLogVersion()));
        storeChannel.position(position.getByteOffset());
        try (PhysicalFlushableChecksumChannel writeChannel = new PhysicalFlushableChecksumChannel(storeChannel, new HeapScopedBuffer(100, INSTANCE))) {
            new LogEntryWriter<>(writeChannel, KernelVersion.LATEST).serialize(dropTransaction);
        }
    }
}

Example 10 with LogFile

use of org.neo4j.kernel.impl.transaction.log.files.LogFile in project neo4j by neo4j.

the class DetachedLogTailScanner method findLogTail.

@Override
protected LogTailInformation findLogTail() {
    LogFile logFile = logFiles.getLogFile();
    long highestLogVersion = logFile.getHighestLogVersion();
    long lowestLogVersion = logFile.getLowestLogVersion();
    try {
        var lastAccessibleCheckpoint = checkPointFile.findLatestCheckpoint();
        if (lastAccessibleCheckpoint.isEmpty()) {
            return noCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion);
        }
        var checkpoint = lastAccessibleCheckpoint.get();
        // found checkpoint pointing to existing position in existing log file
        if (isValidCheckpoint(logFile, checkpoint)) {
            return validCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion, checkpoint);
        }
        if (failOnCorruptedLogFiles) {
            var exceptionMessage = format("Last available %s checkpoint does not point to a valid location in transaction logs.", checkpoint);
            throwUnableToCleanRecover(new RuntimeException(exceptionMessage));
        }
        // our last checkpoint is not valid (we have a pointer to non existent place) lets try to find last one that looks correct
        List<CheckpointInfo> checkpointInfos = checkPointFile.reachableCheckpoints();
        // we know that last one is not valid so no reason to double check that again
        ListIterator<CheckpointInfo> reverseCheckpoints = checkpointInfos.listIterator(checkpointInfos.size() - 1);
        while (reverseCheckpoints.hasPrevious()) {
            CheckpointInfo previousCheckpoint = reverseCheckpoints.previous();
            if (isValidCheckpoint(logFile, previousCheckpoint)) {
                return validCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion, previousCheckpoint);
            }
        }
        // we did not found any valid, we need to restore from the start if possible
        return noCheckpointLogTail(logFile, highestLogVersion, lowestLogVersion);
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
}