Search in sources :

Example 1 with Visitor

use of org.neo4j.internal.helpers.collection.Visitor in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method writeSomeDataWithVersion.

private void writeSomeDataWithVersion(Path file, Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException> visitor, KernelVersion version) throws IOException {
    try (LogVersionedStoreChannel versionedStoreChannel = new PhysicalLogVersionedStoreChannel(fileSystem.write(file), logVersion, CURRENT_LOG_FORMAT_VERSION, file, EMPTY_ACCESSOR);
        PositionAwarePhysicalFlushableChecksumChannel writableLogChannel = new PositionAwarePhysicalFlushableChecksumChannel(versionedStoreChannel, new HeapScopedBuffer(1, KibiByte, INSTANCE))) {
        writeLogHeader(writableLogChannel, new LogHeader(logVersion, 2L, StoreId.UNKNOWN));
        writableLogChannel.beginChecksum();
        Consumer<LogPositionMarker> consumer = marker -> {
            try {
                writableLogChannel.getCurrentPosition(marker);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        LogEntryWriter first = new LogEntryWriter(writableLogChannel, version);
        visitor.visit(Pair.of(first, consumer));
    }
}
Also used : HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) Visitor(org.neo4j.internal.helpers.collection.Visitor) BeforeEach(org.junit.jupiter.api.BeforeEach) LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) MutableInt(org.apache.commons.lang3.mutable.MutableInt) Log(org.neo4j.logging.Log) DatabaseIdFactory.from(org.neo4j.kernel.database.DatabaseIdFactory.from) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) LogFilesBuilder(org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder) Config(org.neo4j.configuration.Config) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) LogHeaderWriter.writeLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) LogCheckPointEvent(org.neo4j.kernel.impl.transaction.tracing.LogCheckPointEvent) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) Monitors(org.neo4j.monitoring.Monitors) BASE_TX_CHECKSUM(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_CHECKSUM) PageCacheTracer(org.neo4j.io.pagecache.tracing.PageCacheTracer) LogVersionRepository(org.neo4j.storageengine.api.LogVersionRepository) PositionAwarePhysicalFlushableChecksumChannel(org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChecksumChannel) Path(java.nio.file.Path) EMPTY_CHECKER(org.neo4j.kernel.recovery.RecoveryStartupChecker.EMPTY_CHECKER) StorageEngine(org.neo4j.storageengine.api.StorageEngine) NO_RECOVERY_REQUIRED(org.neo4j.kernel.recovery.RecoveryStartInformation.NO_RECOVERY_REQUIRED) TestDirectory(org.neo4j.test.rule.TestDirectory) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) LogPositionMarker(org.neo4j.kernel.impl.transaction.log.LogPositionMarker) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) Instant(java.time.Instant) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) Test(org.junit.jupiter.api.Test) CommittedTransactionRepresentation(org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation) INSTANCE(org.neo4j.memory.EmptyMemoryTracker.INSTANCE) TestLogEntryReader.logEntryReader(org.neo4j.kernel.impl.transaction.log.TestLogEntryReader.logEntryReader) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) DatabaseStartAbortedException(org.neo4j.dbms.database.DatabaseStartAbortedException) Mockito.mock(org.mockito.Mockito.mock) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) CsvSource(org.junit.jupiter.params.provider.CsvSource) LogFiles(org.neo4j.kernel.impl.transaction.log.files.LogFiles) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) ProgressReporter(org.neo4j.common.ProgressReporter) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) KernelVersion(org.neo4j.kernel.KernelVersion) ArrayUtils(org.apache.commons.lang3.ArrayUtils) NULL(org.neo4j.io.pagecache.tracing.PageCacheTracer.NULL) DatabaseStartupController(org.neo4j.kernel.database.DatabaseStartupController) LogFile(org.neo4j.kernel.impl.transaction.log.files.LogFile) EMPTY_ACCESSOR(org.neo4j.kernel.impl.transaction.log.files.ChannelNativeAccessor.EMPTY_ACCESSOR) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) Inject(org.neo4j.test.extension.Inject) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) StoreId(org.neo4j.storageengine.api.StoreId) TransactionApplicationMode(org.neo4j.storageengine.api.TransactionApplicationMode) GraphDatabaseInternalSettings(org.neo4j.configuration.GraphDatabaseInternalSettings) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) LogEntryCommit(org.neo4j.kernel.impl.transaction.log.entry.LogEntryCommit) Files(java.nio.file.Files) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) ExceptionUtils.getRootCause(org.apache.commons.lang3.exception.ExceptionUtils.getRootCause) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) CURRENT_LOG_FORMAT_VERSION(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_LOG_FORMAT_VERSION) SimpleLogVersionRepository(org.neo4j.kernel.impl.transaction.SimpleLogVersionRepository) Mockito.verify(org.mockito.Mockito.verify) Consumer(java.util.function.Consumer) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) UUID.randomUUID(java.util.UUID.randomUUID) Mockito.never(org.mockito.Mockito.never) CURRENT_FORMAT_LOG_HEADER_SIZE(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_FORMAT_LOG_HEADER_SIZE) NO_MONITOR(org.neo4j.kernel.recovery.RecoveryStartInformationProvider.NO_MONITOR) HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) Pair(org.neo4j.internal.helpers.collection.Pair) KibiByte(org.neo4j.io.ByteUnit.KibiByte) TransactionIdStore(org.neo4j.storageengine.api.TransactionIdStore) SimpleTransactionIdStore(org.neo4j.kernel.impl.transaction.SimpleTransactionIdStore) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) PositionAwarePhysicalFlushableChecksumChannel(org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChecksumChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) IOException(java.io.IOException) LogHeaderWriter.writeLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) LogPositionMarker(org.neo4j.kernel.impl.transaction.log.LogPositionMarker)

Example 2 with Visitor

use of org.neo4j.internal.helpers.collection.Visitor in project neo4j by neo4j.

the class NodeStoreTest method scanningRecordsShouldVisitEachInUseRecordOnce.

@Test
void scanningRecordsShouldVisitEachInUseRecordOnce() throws IOException {
    // GIVEN we have a NodeStore with data that spans several pages...
    nodeStore = newNodeStore(fs);
    ThreadLocalRandom rng = ThreadLocalRandom.current();
    final MutableLongSet nextRelSet = new LongHashSet();
    for (int i = 0; i < 10_000; i++) {
        // Enough records to span several pages
        int nextRelCandidate = rng.nextInt(0, Integer.MAX_VALUE);
        if (nextRelSet.add(nextRelCandidate)) {
            long nodeId = nodeStore.nextId(NULL);
            NodeRecord record = new NodeRecord(nodeId).initialize(true, 20, false, nextRelCandidate, 0);
            nodeStore.updateRecord(record, NULL);
            if (rng.nextInt(0, 10) < 3) {
                nextRelSet.remove(nextRelCandidate);
                record.setInUse(false);
                nodeStore.updateRecord(record, NULL);
            }
        }
    }
    // ...WHEN we now have an interesting set of node records, and we
    // visit each and remove that node from our nextRelSet...
    Visitor<NodeRecord, IOException> scanner = record -> {
        // ...THEN we should observe that no nextRel is ever removed twice...
        assertTrue(nextRelSet.remove(record.getNextRel()));
        return false;
    };
    nodeStore.scanAllRecords(scanner, NULL);
    // ...NOR do we have anything left in the set afterwards.
    assertTrue(nextRelSet.isEmpty());
}
Also used : Visitor(org.neo4j.internal.helpers.collection.Visitor) IndexedIdGenerator(org.neo4j.internal.id.indexed.IndexedIdGenerator) LongSupplier(java.util.function.LongSupplier) CursorContext(org.neo4j.io.pagecache.context.CursorContext) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) NORMAL(org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL) Config(org.neo4j.configuration.Config) NO_NEXT_RELATIONSHIP(org.neo4j.kernel.impl.store.record.Record.NO_NEXT_RELATIONSHIP) NullLogProvider(org.neo4j.logging.NullLogProvider) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) PageCacheSupportExtension(org.neo4j.test.extension.pagecache.PageCacheSupportExtension) NO_LABELS_FIELD(org.neo4j.kernel.impl.store.record.Record.NO_LABELS_FIELD) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) LongHashSet(org.eclipse.collections.impl.set.mutable.primitive.LongHashSet) Arrays.asList(java.util.Arrays.asList) PageCacheTracer(org.neo4j.io.pagecache.tracing.PageCacheTracer) NULL(org.neo4j.io.pagecache.context.CursorContext.NULL) RecoveryCleanupWorkCollector(org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector) Path(java.nio.file.Path) PageCache(org.neo4j.io.pagecache.PageCache) DatabaseReadOnlyChecker.writable(org.neo4j.configuration.helpers.DatabaseReadOnlyChecker.writable) IOController(org.neo4j.io.pagecache.IOController) DynamicRecord(org.neo4j.kernel.impl.store.record.DynamicRecord) Test(org.junit.jupiter.api.Test) IdType(org.neo4j.internal.id.IdType) EphemeralNeo4jLayoutExtension(org.neo4j.test.extension.EphemeralNeo4jLayoutExtension) INSTANCE(org.neo4j.memory.EmptyMemoryTracker.INSTANCE) DefaultIdGeneratorFactory(org.neo4j.internal.id.DefaultIdGeneratorFactory) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) MutableBoolean(org.apache.commons.lang3.mutable.MutableBoolean) DatabaseReadOnlyChecker(org.neo4j.configuration.helpers.DatabaseReadOnlyChecker) Mockito.mock(org.mockito.Mockito.mock) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) ExceptionUtils.indexOfThrowable(org.apache.commons.lang3.exception.ExceptionUtils.indexOfThrowable) IdGeneratorFactory(org.neo4j.internal.id.IdGeneratorFactory) PagedFile(org.neo4j.io.pagecache.PagedFile) Mockito.spy(org.mockito.Mockito.spy) ArrayList(java.util.ArrayList) DelegatingPageCache(org.neo4j.io.pagecache.DelegatingPageCache) StringContains.containsString(org.hamcrest.core.StringContains.containsString) Inject(org.neo4j.test.extension.Inject) RegisterExtension(org.junit.jupiter.api.extension.RegisterExtension) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) NO_NEXT_PROPERTY(org.neo4j.kernel.impl.store.record.Record.NO_NEXT_PROPERTY) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) DynamicArrayStore.allocateFromNumbers(org.neo4j.kernel.impl.store.DynamicArrayStore.allocateFromNumbers) NodeStore.readOwnerFromDynamicLabelsRecord(org.neo4j.kernel.impl.store.NodeStore.readOwnerFromDynamicLabelsRecord) ReusableRecordsAllocator(org.neo4j.kernel.impl.store.allocator.ReusableRecordsAllocator) OpenOption(java.nio.file.OpenOption) NULL_REFERENCE(org.neo4j.kernel.impl.store.record.Record.NULL_REFERENCE) IOException(java.io.IOException) Mockito.verify(org.mockito.Mockito.verify) MutableLongSet(org.eclipse.collections.api.set.primitive.MutableLongSet) RecoveryCleanupWorkCollector.immediate(org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector.immediate) AfterEach(org.junit.jupiter.api.AfterEach) Mockito.never(org.mockito.Mockito.never) EphemeralFileSystemAbstraction(org.neo4j.io.fs.EphemeralFileSystemAbstraction) NodeRecord(org.neo4j.kernel.impl.store.record.NodeRecord) ImmutableSet(org.eclipse.collections.api.set.ImmutableSet) Collections(java.util.Collections) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) LongHashSet(org.eclipse.collections.impl.set.mutable.primitive.LongHashSet) NodeRecord(org.neo4j.kernel.impl.store.record.NodeRecord) MutableLongSet(org.eclipse.collections.api.set.primitive.MutableLongSet) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) IOException(java.io.IOException) Test(org.junit.jupiter.api.Test)

Aggregations

IOException (java.io.IOException)2 Path (java.nio.file.Path)2 AfterEach (org.junit.jupiter.api.AfterEach)2 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)2 Assertions.assertThrows (org.junit.jupiter.api.Assertions.assertThrows)2 Assertions.assertTrue (org.junit.jupiter.api.Assertions.assertTrue)2 Test (org.junit.jupiter.api.Test)2 ArgumentMatchers.any (org.mockito.ArgumentMatchers.any)2 Mockito.mock (org.mockito.Mockito.mock)2 Mockito.never (org.mockito.Mockito.never)2 Mockito.verify (org.mockito.Mockito.verify)2 Config (org.neo4j.configuration.Config)2 Visitor (org.neo4j.internal.helpers.collection.Visitor)2 DatabaseLayout (org.neo4j.io.layout.DatabaseLayout)2 PageCacheTracer (org.neo4j.io.pagecache.tracing.PageCacheTracer)2 INSTANCE (org.neo4j.memory.EmptyMemoryTracker.INSTANCE)2 Inject (org.neo4j.test.extension.Inject)2 Files (java.nio.file.Files)1 OpenOption (java.nio.file.OpenOption)1 Instant (java.time.Instant)1