Examples with AdversarialFileSystemAbstraction org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction used on opensource projects

Example 11 with AdversarialFileSystemAbstraction

use of org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction in project neo4j by neo4j.

the class SingleFilePageSwapperTest method mustHandleMischiefInPositionedVectoredWrite.

@Test
public void mustHandleMischiefInPositionedVectoredWrite() throws Exception {
    int bytesTotal = 512;
    int bytesPerPage = 32;
    int pageCount = bytesTotal / bytesPerPage;
    byte[] data = new byte[bytesTotal];
    ThreadLocalRandom.current().nextBytes(data);
    ByteBufferPage zeroPage = createPage(bytesPerPage);
    clear(zeroPage);
    File file = getFile();
    PageSwapperFactory factory = createSwapperFactory();
    RandomAdversary adversary = new RandomAdversary(0.5, 0.0, 0.0);
    factory.setFileSystemAbstraction(new AdversarialFileSystemAbstraction(adversary, getFs()));
    PageSwapper swapper = createSwapper(factory, file, bytesPerPage, NO_CALLBACK, true);
    ByteBufferPage[] writePages = new ByteBufferPage[pageCount];
    ByteBufferPage[] readPages = new ByteBufferPage[pageCount];
    ByteBufferPage[] zeroPages = new ByteBufferPage[pageCount];
    for (int i = 0; i < pageCount; i++) {
        writePages[i] = createPage(bytesPerPage);
        writePages[i].putBytes(data, 0, i * bytesPerPage, bytesPerPage);
        readPages[i] = createPage(bytesPerPage);
        zeroPages[i] = zeroPage;
    }
    try {
        for (int i = 0; i < 10_000; i++) {
            adversary.setProbabilityFactor(0);
            swapper.write(0, zeroPages, 0, pageCount);
            adversary.setProbabilityFactor(1);
            swapper.write(0, writePages, 0, pageCount);
            for (ByteBufferPage readPage : readPages) {
                clear(readPage);
            }
            adversary.setProbabilityFactor(0);
            assertThat(swapper.read(0, readPages, 0, pageCount), is((long) bytesTotal));
            for (int j = 0; j < pageCount; j++) {
                assertThat(array(readPages[j].buffer), is(array(writePages[j].buffer)));
            }
        }
    } finally {
        swapper.close();
    }
}

Example 12 with AdversarialFileSystemAbstraction

use of org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction in project neo4j by neo4j.

the class PageCacheSlowTest method pageCacheMustRemainInternallyConsistentWhenGettingRandomFailures.

@RepeatRule.Repeat(times = 1000)
@Test(timeout = LONG_TIMEOUT_MILLIS)
public void pageCacheMustRemainInternallyConsistentWhenGettingRandomFailures() throws Exception {
    // NOTE: This test is inherently non-deterministic. This means that every failure must be
    // thoroughly investigated, since they have a good chance of being a real issue.
    // This is effectively a targeted robustness test.
    RandomAdversary adversary = new RandomAdversary(0.5, 0.2, 0.2);
    adversary.setProbabilityFactor(0.0);
    FileSystemAbstraction fs = new AdversarialFileSystemAbstraction(adversary, this.fs);
    ThreadLocalRandom rng = ThreadLocalRandom.current();
    // Because our test failures are non-deterministic, we use this tracer to capture a full history of the
    // events leading up to any given failure.
    LinearTracers linearTracers = LinearHistoryTracerFactory.pageCacheTracer();
    getPageCache(fs, maxPages, pageCachePageSize, linearTracers.getPageCacheTracer(), linearTracers.getCursorTracerSupplier());
    PagedFile pfA = pageCache.map(existingFile("a"), filePageSize);
    PagedFile pfB = pageCache.map(existingFile("b"), filePageSize / 2 + 1);
    adversary.setProbabilityFactor(1.0);
    for (int i = 0; i < 1000; i++) {
        PagedFile pagedFile = rng.nextBoolean() ? pfA : pfB;
        long maxPageId = pagedFile.getLastPageId();
        boolean performingRead = rng.nextBoolean() && maxPageId != -1;
        long startingPage = maxPageId < 0 ? 0 : rng.nextLong(maxPageId + 1);
        int pf_flags = performingRead ? PF_SHARED_READ_LOCK : PF_SHARED_WRITE_LOCK;
        int pageSize = pagedFile.pageSize();
        try (PageCursor cursor = pagedFile.io(startingPage, pf_flags)) {
            if (performingRead) {
                performConsistentAdversarialRead(cursor, maxPageId, startingPage, pageSize);
            } else {
                performConsistentAdversarialWrite(cursor, rng, pageSize);
            }
        } catch (AssertionError error) {
            // Capture any exception that might have hit the eviction thread.
            adversary.setProbabilityFactor(0.0);
            try (PageCursor cursor = pagedFile.io(0, PF_SHARED_WRITE_LOCK)) {
                for (int j = 0; j < 100; j++) {
                    cursor.next(rng.nextLong(maxPageId + 1));
                }
            } catch (Throwable throwable) {
                error.addSuppressed(throwable);
            }
            throw error;
        } catch (Throwable throwable) {
        // Don't worry about it... it's fine!
        //                throwable.printStackTrace(); // only enable this when debugging test failures.
        }
    }
    // Unmapping will cause pages to be flushed.
    // We don't want that to fail, since it will upset the test tear-down.
    adversary.setProbabilityFactor(0.0);
    try {
        // Flushing all pages, if successful, should clear any internal
        // exception.
        pageCache.flushAndForce();
        // Do some post-chaos verification of what has been written.
        verifyAdversarialPagedContent(pfA);
        verifyAdversarialPagedContent(pfB);
        pfA.close();
        pfB.close();
    } catch (Throwable e) {
        linearTracers.printHistory(System.err);
        throw e;
    }
}

Example 13 with AdversarialFileSystemAbstraction

use of org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction in project neo4j by neo4j.

the class DurableStateStorageIT method shouldProperlyRecoveryAfterCrashOnFileForceDuringWrite.

@Test
public void shouldProperlyRecoveryAfterCrashOnFileForceDuringWrite() throws Exception {
    EphemeralFileSystemAbstraction normalFSA = fileSystemRule.get();
    /*
         * Magic number warning. For a rotation threshold of 14, 990 operations on file A falls on a force() of the
         * current active file. This has been discovered via experimentation. The end result is that there is a
         * flush (but not write) a value. This should be recoverable. Interestingly, the failure semantics are a bit
         * unclear on what should happen to that value. We assume that exception during persistence requires recovery
         * to discover if the last argument made it to disk or not. Since we use an EFSA, force is not necessary and
         * the value that caused the failure is actually "persisted" and recovered.
         */
    AdversarialFileSystemAbstraction breakingFSA = new AdversarialFileSystemAbstraction(new MethodGuardedAdversary(new CountingAdversary(40, true), StoreChannel.class.getMethod("force", boolean.class)), normalFSA);
    SelectiveFileSystemAbstraction combinedFSA = new SelectiveFileSystemAbstraction(new File(new File(testDir.directory(), "long-state"), "long.a"), breakingFSA, normalFSA);
    long lastValue = 0;
    try (LongState persistedState = new LongState(combinedFSA, testDir.directory(), 14)) {
        while (// it will break from the Exception that AFS will throw
        true) {
            long tempValue = lastValue + 1;
            persistedState.setTheState(tempValue);
            lastValue = tempValue;
        }
    } catch (Exception expected) {
        // this stack trace should contain force()
        ensureStackTraceContainsExpectedMethod(expected.getStackTrace(), "force");
    }
    try (LongState restoredState = new LongState(normalFSA, testDir.directory(), 14)) {
        assertThat(restoredState.getTheState(), greaterThanOrEqualTo(lastValue));
    }
}

Example 14 with AdversarialFileSystemAbstraction

use of org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction in project neo4j by neo4j.

the class DurableStateStorageIT method shouldRecoverAfterCrashUnderLoad.

@Test
public void shouldRecoverAfterCrashUnderLoad() throws Exception {
    EphemeralFileSystemAbstraction delegate = fileSystemRule.get();
    AdversarialFileSystemAbstraction fsa = new AdversarialFileSystemAbstraction(new MethodGuardedAdversary(new CountingAdversary(100, true), StoreChannel.class.getMethod("writeAll", ByteBuffer.class)), delegate);
    long lastValue = 0;
    try (LongState persistedState = new LongState(fsa, testDir.directory(), 14)) {
        while (// it will break from the Exception that AFS will throw
        true) {
            long tempValue = lastValue + 1;
            persistedState.setTheState(tempValue);
            lastValue = tempValue;
        }
    } catch (Exception expected) {
        ensureStackTraceContainsExpectedMethod(expected.getStackTrace(), "writeAll");
    }
    try (LongState restoredState = new LongState(delegate, testDir.directory(), 4)) {
        assertEquals(lastValue, restoredState.getTheState());
    }
}

Example 15 with AdversarialFileSystemAbstraction

use of org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction in project neo4j by neo4j.

the class DurableStateStorageIT method shouldProperlyRecoveryAfterCrashingDuringRecovery.

@Test
public void shouldProperlyRecoveryAfterCrashingDuringRecovery() throws Exception {
    EphemeralFileSystemAbstraction normalFSA = fileSystemRule.get();
    long lastValue = 0;
    try (LongState persistedState = new LongState(normalFSA, testDir.directory(), 14)) {
        for (int i = 0; i < 100; i++) {
            long tempValue = lastValue + 1;
            persistedState.setTheState(tempValue);
            lastValue = tempValue;
        }
    }
    try {
        // We create a new state that will attempt recovery. The AFS will make it fail on open() of one of the files
        new LongState(new AdversarialFileSystemAbstraction(new MethodGuardedAdversary(new CountingAdversary(1, true), FileSystemAbstraction.class.getMethod("open", File.class, String.class)), normalFSA), testDir.directory(), 14);
        fail("Should have failed recovery");
    } catch (Exception expected) {
        // this stack trace should contain open()
        ensureStackTraceContainsExpectedMethod(expected.getCause().getStackTrace(), "open");
    }
    // Recovery over the normal filesystem after a failed recovery should proceed correctly
    try (LongState recoveredState = new LongState(normalFSA, testDir.directory(), 14)) {
        assertThat(recoveredState.getTheState(), greaterThanOrEqualTo(lastValue));
    }
}