Search in sources :

Example 11 with HeapScopedBuffer

use of org.neo4j.io.memory.HeapScopedBuffer in project neo4j by neo4j.

the class CorruptedLogsTruncator method backupCorruptedContent.

private void backupCorruptedContent(long recoveredTransactionLogVersion, long recoveredTransactionOffset, Optional<CheckpointInfo> corruptCheckpoint) throws IOException {
    Path corruptedLogArchive = getArchiveFile(recoveredTransactionLogVersion, recoveredTransactionOffset);
    try (ZipOutputStream recoveryContent = new ZipOutputStream(fs.openAsOutputStream(corruptedLogArchive, false));
        var bufferScope = new HeapScopedBuffer(1, MebiByte, memoryTracker)) {
        LogFile transactionLogFile = logFiles.getLogFile();
        copyLogsContent(recoveredTransactionLogVersion, recoveredTransactionOffset, transactionLogFile.getHighestLogVersion(), recoveryContent, bufferScope, transactionLogFile::getLogFileForVersion);
        if (corruptCheckpoint.isPresent()) {
            LogPosition checkpointPosition = corruptCheckpoint.get().getCheckpointEntryPosition();
            CheckpointFile checkpointFile = logFiles.getCheckpointFile();
            copyLogsContent(checkpointPosition.getLogVersion(), checkpointPosition.getByteOffset(), checkpointFile.getCurrentDetachedLogVersion(), recoveryContent, bufferScope, checkpointFile::getDetachedCheckpointFileForVersion);
        }
    }
}
Also used : Path(java.nio.file.Path) HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) LogFile(org.neo4j.kernel.impl.transaction.log.files.LogFile) ZipOutputStream(java.util.zip.ZipOutputStream) CheckpointFile(org.neo4j.kernel.impl.transaction.log.files.checkpoint.CheckpointFile) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition)

Example 12 with HeapScopedBuffer

use of org.neo4j.io.memory.HeapScopedBuffer in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method writeSomeDataWithVersion.

private void writeSomeDataWithVersion(Path file, Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException> visitor, KernelVersion version) throws IOException {
    try (LogVersionedStoreChannel versionedStoreChannel = new PhysicalLogVersionedStoreChannel(fileSystem.write(file), logVersion, CURRENT_LOG_FORMAT_VERSION, file, EMPTY_ACCESSOR);
        PositionAwarePhysicalFlushableChecksumChannel writableLogChannel = new PositionAwarePhysicalFlushableChecksumChannel(versionedStoreChannel, new HeapScopedBuffer(1, KibiByte, INSTANCE))) {
        writeLogHeader(writableLogChannel, new LogHeader(logVersion, 2L, StoreId.UNKNOWN));
        writableLogChannel.beginChecksum();
        Consumer<LogPositionMarker> consumer = marker -> {
            try {
                writableLogChannel.getCurrentPosition(marker);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        LogEntryWriter first = new LogEntryWriter(writableLogChannel, version);
        visitor.visit(Pair.of(first, consumer));
    }
}
Also used : HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) Visitor(org.neo4j.internal.helpers.collection.Visitor) BeforeEach(org.junit.jupiter.api.BeforeEach) LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) MutableInt(org.apache.commons.lang3.mutable.MutableInt) Log(org.neo4j.logging.Log) DatabaseIdFactory.from(org.neo4j.kernel.database.DatabaseIdFactory.from) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) LogFilesBuilder(org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder) Config(org.neo4j.configuration.Config) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) LogHeaderWriter.writeLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) LogCheckPointEvent(org.neo4j.kernel.impl.transaction.tracing.LogCheckPointEvent) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) Monitors(org.neo4j.monitoring.Monitors) BASE_TX_CHECKSUM(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_CHECKSUM) PageCacheTracer(org.neo4j.io.pagecache.tracing.PageCacheTracer) LogVersionRepository(org.neo4j.storageengine.api.LogVersionRepository) PositionAwarePhysicalFlushableChecksumChannel(org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChecksumChannel) Path(java.nio.file.Path) EMPTY_CHECKER(org.neo4j.kernel.recovery.RecoveryStartupChecker.EMPTY_CHECKER) StorageEngine(org.neo4j.storageengine.api.StorageEngine) NO_RECOVERY_REQUIRED(org.neo4j.kernel.recovery.RecoveryStartInformation.NO_RECOVERY_REQUIRED) TestDirectory(org.neo4j.test.rule.TestDirectory) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) LogPositionMarker(org.neo4j.kernel.impl.transaction.log.LogPositionMarker) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) Instant(java.time.Instant) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) Test(org.junit.jupiter.api.Test) CommittedTransactionRepresentation(org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation) INSTANCE(org.neo4j.memory.EmptyMemoryTracker.INSTANCE) TestLogEntryReader.logEntryReader(org.neo4j.kernel.impl.transaction.log.TestLogEntryReader.logEntryReader) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) DatabaseStartAbortedException(org.neo4j.dbms.database.DatabaseStartAbortedException) Mockito.mock(org.mockito.Mockito.mock) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) CsvSource(org.junit.jupiter.params.provider.CsvSource) LogFiles(org.neo4j.kernel.impl.transaction.log.files.LogFiles) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) ProgressReporter(org.neo4j.common.ProgressReporter) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) KernelVersion(org.neo4j.kernel.KernelVersion) ArrayUtils(org.apache.commons.lang3.ArrayUtils) NULL(org.neo4j.io.pagecache.tracing.PageCacheTracer.NULL) DatabaseStartupController(org.neo4j.kernel.database.DatabaseStartupController) LogFile(org.neo4j.kernel.impl.transaction.log.files.LogFile) EMPTY_ACCESSOR(org.neo4j.kernel.impl.transaction.log.files.ChannelNativeAccessor.EMPTY_ACCESSOR) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) Inject(org.neo4j.test.extension.Inject) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) StoreId(org.neo4j.storageengine.api.StoreId) TransactionApplicationMode(org.neo4j.storageengine.api.TransactionApplicationMode) GraphDatabaseInternalSettings(org.neo4j.configuration.GraphDatabaseInternalSettings) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) LogEntryCommit(org.neo4j.kernel.impl.transaction.log.entry.LogEntryCommit) Files(java.nio.file.Files) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) ExceptionUtils.getRootCause(org.apache.commons.lang3.exception.ExceptionUtils.getRootCause) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) CURRENT_LOG_FORMAT_VERSION(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_LOG_FORMAT_VERSION) SimpleLogVersionRepository(org.neo4j.kernel.impl.transaction.SimpleLogVersionRepository) Mockito.verify(org.mockito.Mockito.verify) Consumer(java.util.function.Consumer) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) UUID.randomUUID(java.util.UUID.randomUUID) Mockito.never(org.mockito.Mockito.never) CURRENT_FORMAT_LOG_HEADER_SIZE(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_FORMAT_LOG_HEADER_SIZE) NO_MONITOR(org.neo4j.kernel.recovery.RecoveryStartInformationProvider.NO_MONITOR) HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) Pair(org.neo4j.internal.helpers.collection.Pair) KibiByte(org.neo4j.io.ByteUnit.KibiByte) TransactionIdStore(org.neo4j.storageengine.api.TransactionIdStore) SimpleTransactionIdStore(org.neo4j.kernel.impl.transaction.SimpleTransactionIdStore) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) PositionAwarePhysicalFlushableChecksumChannel(org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChecksumChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) IOException(java.io.IOException) LogHeaderWriter.writeLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) LogPositionMarker(org.neo4j.kernel.impl.transaction.log.LogPositionMarker)

Example 13 with HeapScopedBuffer

use of org.neo4j.io.memory.HeapScopedBuffer in project neo4j by neo4j.

the class TransactionLogFile method hasAnyEntries.

@Override
public boolean hasAnyEntries(long version) {
    try {
        Path logFile = getLogFileForVersion(version);
        var logHeader = extractHeader(version, false);
        if (logHeader == null) {
            return false;
        }
        int headerSize = Math.toIntExact(logHeader.getStartPosition().getByteOffset());
        if (fileSystem.getFileSize(logFile) <= headerSize) {
            return false;
        }
        try (StoreChannel channel = fileSystem.read(logFile)) {
            try (var scopedBuffer = new HeapScopedBuffer(headerSize + 1, context.getMemoryTracker())) {
                var buffer = scopedBuffer.getBuffer();
                channel.readAll(buffer);
                buffer.flip();
                return buffer.get(headerSize) != 0;
            }
        }
    } catch (IOException e) {
        return false;
    }
}
Also used : Path(java.nio.file.Path) HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) StoreChannel(org.neo4j.io.fs.StoreChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) DelegatingStoreChannel(org.neo4j.io.fs.DelegatingStoreChannel) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException)

Example 14 with HeapScopedBuffer

use of org.neo4j.io.memory.HeapScopedBuffer in project neo4j by neo4j.

the class TransactionLogChannelAllocator method openLogChannel.

public PhysicalLogVersionedStoreChannel openLogChannel(long version, boolean raw) throws IOException {
    Path fileToOpen = fileHelper.getLogFileForVersion(version);
    if (!fileSystem.fileExists(fileToOpen)) {
        throw new NoSuchFileException(fileToOpen.toAbsolutePath().toString());
    }
    StoreChannel rawChannel = null;
    try {
        rawChannel = fileSystem.read(fileToOpen);
        try (var scopedBuffer = new HeapScopedBuffer(CURRENT_FORMAT_LOG_HEADER_SIZE, logFilesContext.getMemoryTracker())) {
            var buffer = scopedBuffer.getBuffer();
            LogHeader header = readLogHeader(buffer, rawChannel, true, fileToOpen);
            if ((header == null) || (header.getLogVersion() != version)) {
                throw new IllegalStateException(format("Unexpected log file header. Expected header version: %d, actual header: %s", version, header != null ? header.toString() : "null header."));
            }
            var versionedStoreChannel = new PhysicalLogVersionedStoreChannel(rawChannel, version, header.getLogFormatVersion(), fileToOpen, nativeChannelAccessor, raw);
            if (!raw) {
                nativeChannelAccessor.adviseSequentialAccessAndKeepInCache(rawChannel, version);
            }
            return versionedStoreChannel;
        }
    } catch (NoSuchFileException cause) {
        throw (NoSuchFileException) new NoSuchFileException(fileToOpen.toAbsolutePath().toString()).initCause(cause);
    } catch (Throwable unexpectedError) {
        if (rawChannel != null) {
            // If we managed to open the file before failing, then close the channel
            try {
                rawChannel.close();
            } catch (IOException e) {
                unexpectedError.addSuppressed(e);
            }
        }
        throw unexpectedError;
    }
}
Also used : Path(java.nio.file.Path) HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) StoreChannel(org.neo4j.io.fs.StoreChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) NoSuchFileException(java.nio.file.NoSuchFileException) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) IOException(java.io.IOException) LogHeaderReader.readLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader)

Example 15 with HeapScopedBuffer

use of org.neo4j.io.memory.HeapScopedBuffer in project neo4j by neo4j.

the class AbstractLogTailScanner method verifyNoMoreReadableDataAvailable.

private void verifyNoMoreReadableDataAvailable(long version, LogVersionedStoreChannel channel, LogPosition logPosition, long channelLeftovers) throws IOException {
    long initialPosition = channel.position();
    try {
        channel.position(logPosition.getByteOffset());
        try (var scopedBuffer = new HeapScopedBuffer(safeCastLongToInt(min(kibiBytes(12), channelLeftovers)), memoryTracker)) {
            ByteBuffer byteBuffer = scopedBuffer.getBuffer();
            channel.readAll(byteBuffer);
            byteBuffer.flip();
            if (!isAllZerosBuffer(byteBuffer)) {
                throw new RuntimeException(format("Transaction log files with version %d has some data available after last readable log entry. " + "Last readable position %d, read ahead buffer content: %s.", version, logPosition.getByteOffset(), dumpBufferToString(byteBuffer)));
            }
        }
    } finally {
        channel.position(initialPosition);
    }
}
Also used : HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) ByteBuffer(java.nio.ByteBuffer)

Aggregations

HeapScopedBuffer (org.neo4j.io.memory.HeapScopedBuffer)20 StoreChannel (org.neo4j.io.fs.StoreChannel)10 Test (org.junit.jupiter.api.Test)9 Path (java.nio.file.Path)8 PhysicalFlushableChecksumChannel (org.neo4j.io.fs.PhysicalFlushableChecksumChannel)6 ByteBuffer (java.nio.ByteBuffer)5 IOException (java.io.IOException)4 PhysicalLogVersionedStoreChannel (org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel)4 InputStream (java.io.InputStream)3 LogPosition (org.neo4j.kernel.impl.transaction.log.LogPosition)3 LogHeader (org.neo4j.kernel.impl.transaction.log.entry.LogHeader)3 LogFile (org.neo4j.kernel.impl.transaction.log.files.LogFile)3 Checksum (java.util.zip.Checksum)2 LogVersionedStoreChannel (org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel)2 StoreId (org.neo4j.storageengine.api.StoreId)2 UncheckedIOException (java.io.UncheckedIOException)1 Files (java.nio.file.Files)1 NoSuchFileException (java.nio.file.NoSuchFileException)1 Instant (java.time.Instant)1 UUID.randomUUID (java.util.UUID.randomUUID)1