Examples with PageCache org.neo4j.io.pagecache.PageCache used on opensource projects

Example 86 with PageCache

use of org.neo4j.io.pagecache.PageCache in project neo4j by neo4j.

the class GBPTreeRecoveryTest method doShouldRecoverFromAnything.

private void doShouldRecoverFromAnything(boolean replayRecoveryExactlyFromCheckpoint) throws Exception {
    assertInitialized();
    // GIVEN
    // a tree which has had random updates and checkpoints in it, load generated with specific seed
    File file = directory.file("index");
    List<Action> load = generateLoad();
    int lastCheckPointIndex = indexOfLastCheckpoint(load);
    {
        // _,_,_,_,_,_,_,c,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,c,_,_,_,_,_,_,_,_,_,_,_
        //                                                 ^             ^
        //                                                 |             |------------ crash flush index
        //                                                 |-------------------------- last checkpoint index
        //
        PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file);
        // Execute all actions up to and including last checkpoint ...
        execute(load.subList(0, lastCheckPointIndex + 1), index);
        // ... a random amount of the remaining "unsafe" actions ...
        int numberOfRemainingActions = load.size() - lastCheckPointIndex - 1;
        int crashFlushIndex = lastCheckPointIndex + random.nextInt(numberOfRemainingActions) + 1;
        execute(load.subList(lastCheckPointIndex + 1, crashFlushIndex), index);
        // ... flush ...
        pageCache.flushAndForce();
        // ... execute the remaining actions
        execute(load.subList(crashFlushIndex, load.size()), index);
        // ... and finally crash
        fs.snapshot(throwing(() -> {
            index.close();
            pageCache.close();
        }));
    }
    // WHEN doing recovery
    List<Action> recoveryActions;
    if (replayRecoveryExactlyFromCheckpoint) {
        recoveryActions = recoveryActions(load, lastCheckPointIndex + 1);
    } else {
        recoveryActions = recoveryActions(load, random.nextInt(lastCheckPointIndex + 1));
    }
    // first crashing during recovery
    int numberOfCrashesDuringRecovery = random.intBetween(0, 3);
    for (int i = 0; i < numberOfCrashesDuringRecovery; i++) {
        PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file);
        int numberOfActionsToRecoverBeforeCrashing = random.intBetween(1, recoveryActions.size());
        recover(recoveryActions.subList(0, numberOfActionsToRecoverBeforeCrashing), index);
        pageCache.flushAndForce();
        fs.snapshot(throwing(() -> {
            index.close();
            pageCache.close();
        }));
    }
    // to finally apply all actions after last checkpoint and verify tree
    try (PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file)) {
        recover(recoveryActions, index);
        index.finishRecovery();
        // THEN
        // we should end up with a consistent index containing all the stuff load says
        index.consistencyCheck();
        long[] /*key,value,key,value...*/
        aggregate = expectedSortedAggregatedDataFromGeneratedLoad(load);
        try (RawCursor<Hit<MutableLong, MutableLong>, IOException> cursor = index.seek(new MutableLong(Long.MIN_VALUE), new MutableLong(Long.MAX_VALUE))) {
            for (int i = 0; i < aggregate.length; ) {
                assertTrue(cursor.next());
                Hit<MutableLong, MutableLong> hit = cursor.get();
                assertEquals(aggregate[i++], hit.key().longValue());
                assertEquals(aggregate[i++], hit.value().longValue());
            }
            assertFalse(cursor.next());
        }
    }
}

Example 87 with PageCache

use of org.neo4j.io.pagecache.PageCache in project neo4j by neo4j.

the class GBPTreeRecoveryTest method shouldRecoverFromCrashBeforeFirstCheckpoint.

@Test
public void shouldRecoverFromCrashBeforeFirstCheckpoint() throws Exception {
    // GIVEN
    // a tree with only small amount of data that has not yet seen checkpoint from outside
    File file = directory.file("index");
    {
        PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file);
        Writer<MutableLong, MutableLong> writer = index.writer();
        key.setValue(1L);
        value.setValue(10L);
        writer.put(key, value);
        pageCache.flushAndForce();
        fs.snapshot(throwing(() -> {
            writer.close();
            index.close();
            pageCache.close();
        }));
    }
    // WHEN
    try (PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file)) {
        // this is the mimic:ed recovery
        index.prepareForRecovery();
        index.finishRecovery();
        try (Writer<MutableLong, MutableLong> writer = index.writer()) {
            writer.put(key, value);
        }
        // THEN
        // we should end up with a consistent index
        index.consistencyCheck();
        // ... containing all the stuff load says
        try (RawCursor<Hit<MutableLong, MutableLong>, IOException> cursor = index.seek(new MutableLong(Long.MIN_VALUE), new MutableLong(Long.MAX_VALUE))) {
            assertTrue(cursor.next());
            Hit<MutableLong, MutableLong> hit = cursor.get();
            assertEquals(key.getValue(), hit.key().getValue());
            assertEquals(value.getValue(), hit.value().getValue());
        }
    }
}

Example 88 with PageCache

use of org.neo4j.io.pagecache.PageCache in project neo4j by neo4j.

the class CrashGenerationCleanerTest method setupPagedFile.

@Before
public void setupPagedFile() throws IOException {
    PageCache pageCache = pageCacheRule.getPageCache(fileSystemRule.get(), config().withPageSize(PAGE_SIZE).withAccessChecks(true));
    pagedFile = pageCache.map(testDirectory.file(FILE_NAME), PAGE_SIZE, CREATE, DELETE_ON_CLOSE);
}

Example 89 with PageCache

use of org.neo4j.io.pagecache.PageCache in project neo4j by neo4j.

the class AccessCheckingPageCacheTest method getPageCursor.

@Before
public void getPageCursor() throws IOException {
    PageCache mockedPageCache = mock(PageCache.class);
    PagedFile mockedPagedFile = mock(PagedFile.class);
    PageCursor mockedCursor = mock(PageCursor.class);
    when(mockedPagedFile.io(anyLong(), anyInt())).thenReturn(mockedCursor);
    when(mockedPageCache.map(any(File.class), anyInt(), anyVararg())).thenReturn(mockedPagedFile);
    pageCache = new AccessCheckingPageCache(mockedPageCache);
    PagedFile file = pageCache.map(new File("some file"), 512);
    cursor = file.io(0, PagedFile.PF_SHARED_READ_LOCK);
}

Example 90 with PageCache

use of org.neo4j.io.pagecache.PageCache in project neo4j by neo4j.

the class CoreBootstrapperTest method shouldSetAllCoreState.

@Test
public void shouldSetAllCoreState() throws Exception {
    // given
    int nodeCount = 100;
    FileSystemAbstraction fileSystem = fileSystemRule.get();
    File classicNeo4jStore = RestoreClusterUtils.createClassicNeo4jStore(testDirectory.directory(), fileSystem, nodeCount, Standard.LATEST_NAME);
    PageCache pageCache = pageCacheRule.getPageCache(fileSystem);
    CoreBootstrapper bootstrapper = new CoreBootstrapper(classicNeo4jStore, pageCache, fileSystem, Config.defaults(), NullLogProvider.getInstance());
    // when
    Set<MemberId> membership = asSet(randomMember(), randomMember(), randomMember());
    CoreSnapshot snapshot = bootstrapper.bootstrap(membership);
    // then
    assertEquals(nodeCount, ((IdAllocationState) snapshot.get(CoreStateType.ID_ALLOCATION)).firstUnallocated(IdType.NODE));
    /* Bootstrapped state is created in RAFT land at index -1 and term -1. */
    assertEquals(0, snapshot.prevIndex());
    assertEquals(0, snapshot.prevTerm());
    /* Lock is initially not taken. */
    assertEquals(new ReplicatedLockTokenState(), snapshot.get(CoreStateType.LOCK_TOKEN));
    /* Raft has the bootstrapped set of members initially. */
    assertEquals(membership, ((RaftCoreState) snapshot.get(CoreStateType.RAFT_CORE_STATE)).committed().members());
    /* The session state is initially empty. */
    assertEquals(new GlobalSessionTrackerState(), snapshot.get(CoreStateType.SESSION_TRACKER));
    LastCommittedIndexFinder lastCommittedIndexFinder = new LastCommittedIndexFinder(new ReadOnlyTransactionIdStore(pageCache, classicNeo4jStore), new ReadOnlyTransactionStore(pageCache, fileSystem, classicNeo4jStore, new Monitors()), NullLogProvider.getInstance());
    long lastCommittedIndex = lastCommittedIndexFinder.getLastCommittedIndex();
    assertEquals(-1, lastCommittedIndex);
}