Search in sources :

Example 16 with PageSwapperFactory

use of org.neo4j.io.pagecache.PageSwapperFactory in project neo4j by neo4j.

the class RandomPageCacheTestHarness method runIteration.

@SuppressWarnings("unchecked")
private void runIteration(long timeout, TimeUnit unit) throws Exception {
    assert filePageSize % recordFormat.getRecordSize() == 0 : "File page size must be a multiple of the record size";
    if (!fixedRandomSeed) {
        randomSeed = ThreadLocalRandom.current().nextLong();
    }
    FileSystemAbstraction fs = this.fs;
    Path[] files = buildFileNames();
    RandomAdversary adversary = new RandomAdversary(mischiefRate, failureRate, errorRate);
    adversary.setProbabilityFactor(0.0);
    if (useAdversarialIO) {
        adversary.setSeed(randomSeed);
        fs = new AdversarialFileSystemAbstraction(adversary, fs);
    }
    PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
    JobScheduler jobScheduler = new ThreadPoolJobScheduler();
    MuninnPageCache cache = new MuninnPageCache(swapperFactory, jobScheduler, MuninnPageCache.config(cachePageCount).pageCacheTracer(tracer));
    if (filePageSize == 0) {
        filePageSize = cache.pageSize();
    }
    cache.setPrintExceptionsOnClose(false);
    Map<Path, PagedFile> fileMap = new HashMap<>(files.length);
    for (int i = 0; i < Math.min(files.length, initialMappedFiles); i++) {
        Path file = files[i];
        fileMap.put(file, cache.map(file, filePageSize, DEFAULT_DATABASE_NAME));
    }
    plan = plan(cache, files, fileMap);
    AtomicBoolean stopSignal = new AtomicBoolean();
    Callable<Void> planRunner = new PlanRunner(plan, stopSignal, profiler);
    Future<Void>[] futures = new Future[concurrencyLevel];
    for (int i = 0; i < concurrencyLevel; i++) {
        futures[i] = executorService.submit(planRunner);
    }
    if (preparation != null) {
        preparation.run(cache, this.fs, plan.getFilesTouched());
    }
    adversary.setProbabilityFactor(1.0);
    plan.start();
    long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout);
    long now;
    try {
        for (Future<Void> future : futures) {
            now = System.currentTimeMillis();
            if (deadlineMillis < now) {
                throw new TimeoutException();
            }
            future.get(deadlineMillis - now, TimeUnit.MILLISECONDS);
        }
        adversary.setProbabilityFactor(0.0);
        runVerificationPhase(cache);
    } finally {
        stopSignal.set(true);
        adversary.setProbabilityFactor(0.0);
        try {
            for (Future<Void> future : futures) {
                future.get(10, TimeUnit.SECONDS);
            }
        } catch (InterruptedException | TimeoutException e) {
            for (Future<Void> future : futures) {
                future.cancel(true);
            }
            throw new RuntimeException(e);
        }
        try {
            plan.close();
            cache.close();
            jobScheduler.close();
            if (this.fs instanceof EphemeralFileSystemAbstraction) {
                this.fs.close();
                this.fs = new EphemeralFileSystemAbstraction();
            } else {
                for (Path file : files) {
                    Files.delete(file);
                }
            }
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
}
Also used : AdversarialFileSystemAbstraction(org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction) EphemeralFileSystemAbstraction(org.neo4j.io.fs.EphemeralFileSystemAbstraction) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) HashMap(java.util.HashMap) EphemeralFileSystemAbstraction(org.neo4j.io.fs.EphemeralFileSystemAbstraction) UncheckedIOException(java.io.UncheckedIOException) MuninnPageCache(org.neo4j.io.pagecache.impl.muninn.MuninnPageCache) SingleFilePageSwapperFactory(org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory) ThreadPoolJobScheduler(org.neo4j.test.scheduler.ThreadPoolJobScheduler) TimeoutException(java.util.concurrent.TimeoutException) Path(java.nio.file.Path) JobScheduler(org.neo4j.scheduler.JobScheduler) ThreadPoolJobScheduler(org.neo4j.test.scheduler.ThreadPoolJobScheduler) PageSwapperFactory(org.neo4j.io.pagecache.PageSwapperFactory) SingleFilePageSwapperFactory(org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory) PagedFile(org.neo4j.io.pagecache.PagedFile) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) RandomAdversary(org.neo4j.adversaries.RandomAdversary) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Future(java.util.concurrent.Future) AdversarialFileSystemAbstraction(org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction)

Example 17 with PageSwapperFactory

use of org.neo4j.io.pagecache.PageSwapperFactory in project neo4j by neo4j.

the class SingleFilePageSwapperTest method doNotReportExternalIoOnCheckpointerCalledVectoredFlush.

@Test
void doNotReportExternalIoOnCheckpointerCalledVectoredFlush() throws IOException {
    createEmptyFile();
    PageSwapperFactory factory = createSwapperFactory(getFs());
    CountingIOController controller = new CountingIOController();
    try (var swapper = createSwapper(factory, getPath(), 4, null, false, false, true, controller)) {
        long target1 = createPage(4);
        long target2 = createPage(4);
        long target3 = createPage(4);
        int numberOfReads = 12;
        int buffers = 3;
        for (int i = 0; i < numberOfReads; i++) {
            assertEquals(12, swapper.write(0, new long[] { target1, target2, target3 }, new int[] { 4, 4, 4 }, buffers, buffers));
        }
        assertEquals(0, controller.getExternalIOCounter());
    }
}
Also used : PageSwapperFactory(org.neo4j.io.pagecache.PageSwapperFactory) PageSwapperTest(org.neo4j.io.pagecache.PageSwapperTest) Test(org.junit.jupiter.api.Test)

Example 18 with PageSwapperFactory

use of org.neo4j.io.pagecache.PageSwapperFactory in project neo4j by neo4j.

the class SingleFilePageSwapperTest method uninterruptibleWrite.

@Test
void uninterruptibleWrite() throws Exception {
    byte[] pageContent = new byte[] { 1, 2, 3, 4 };
    StoreChannel channel = getFs().write(getPath());
    channel.writeAll(wrap(pageContent));
    channel.close();
    PageSwapperFactory factory = createSwapperFactory(getFs());
    PageSwapper swapper = createSwapper(factory, getPath(), 4, null, false);
    long target = createPage(4);
    CountDownLatch startInterruptsLatch = new CountDownLatch(1);
    AtomicBoolean writeFlag = new AtomicBoolean(true);
    var uninterruptibleFuture = operationExecutor.submit(() -> {
        startInterruptsLatch.countDown();
        while (writeFlag.get()) {
            try {
                swapper.write(0, target);
            } catch (Throwable t) {
                throw new RuntimeException(t);
            }
        }
    });
    startInterruptsLatch.await();
    assertFalse(threadRegistryFactory.getThreads().isEmpty());
    for (int i = 0; i < INTERRUPT_ATTEMPTS; i++) {
        threadRegistryFactory.getThreads().forEach(Thread::interrupt);
        parkNanos(MILLISECONDS.toNanos(10));
    }
    writeFlag.set(false);
    assertDoesNotThrow((ThrowingSupplier<?>) uninterruptibleFuture::get);
}
Also used : PageSwapperFactory(org.neo4j.io.pagecache.PageSwapperFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StoreChannel(org.neo4j.io.fs.StoreChannel) DelegatingStoreChannel(org.neo4j.io.fs.DelegatingStoreChannel) PageSwapper(org.neo4j.io.pagecache.PageSwapper) CountDownLatch(java.util.concurrent.CountDownLatch) PageSwapperTest(org.neo4j.io.pagecache.PageSwapperTest) Test(org.junit.jupiter.api.Test)

Example 19 with PageSwapperFactory

use of org.neo4j.io.pagecache.PageSwapperFactory in project neo4j by neo4j.

the class SingleFilePageSwapperTest method reportExternalIoOnSwapOutWithLength.

@Test
void reportExternalIoOnSwapOutWithLength() throws IOException {
    createEmptyFile();
    PageSwapperFactory factory = createSwapperFactory(getFs());
    CountingIOController controller = new CountingIOController();
    try (var swapper = createSwapper(factory, getPath(), 4, null, false, false, true, controller)) {
        long target = createPage(4);
        int numberOfWrites = 42;
        for (int i = 0; i < numberOfWrites; i++) {
            assertEquals(4, swapper.write(0, target, 4));
        }
        assertEquals(numberOfWrites, controller.getExternalIOCounter());
    }
}
Also used : PageSwapperFactory(org.neo4j.io.pagecache.PageSwapperFactory) PageSwapperTest(org.neo4j.io.pagecache.PageSwapperTest) Test(org.junit.jupiter.api.Test)

Example 20 with PageSwapperFactory

use of org.neo4j.io.pagecache.PageSwapperFactory in project neo4j by neo4j.

the class SingleFilePageSwapperTest method mustZeroFillPageBeyondEndOfFile.

@Test
void mustZeroFillPageBeyondEndOfFile() throws Exception {
    byte[] bytes = new byte[] { // --- page 0:
    1, 2, 3, 4, // --- page 1:
    5, 6 };
    StoreChannel channel = getFs().write(getPath());
    channel.writeAll(wrap(bytes));
    channel.close();
    PageSwapperFactory factory = createSwapperFactory(getFs());
    PageSwapper swapper = createSwapper(factory, getPath(), 4, null, false);
    long target = createPage(4);
    assertEquals(2, swapper.read(1, target));
    assertThat(array(target)).containsExactly(5, 6, 0, 0);
}
Also used : PageSwapperFactory(org.neo4j.io.pagecache.PageSwapperFactory) StoreChannel(org.neo4j.io.fs.StoreChannel) DelegatingStoreChannel(org.neo4j.io.fs.DelegatingStoreChannel) PageSwapper(org.neo4j.io.pagecache.PageSwapper) PageSwapperTest(org.neo4j.io.pagecache.PageSwapperTest) Test(org.junit.jupiter.api.Test)

Aggregations

PageSwapperFactory (org.neo4j.io.pagecache.PageSwapperFactory)27 Test (org.junit.jupiter.api.Test)23 PageSwapperTest (org.neo4j.io.pagecache.PageSwapperTest)22 DelegatingStoreChannel (org.neo4j.io.fs.DelegatingStoreChannel)13 StoreChannel (org.neo4j.io.fs.StoreChannel)13 PageSwapper (org.neo4j.io.pagecache.PageSwapper)12 Path (java.nio.file.Path)11 DisabledOnOs (org.junit.jupiter.api.condition.DisabledOnOs)6 RandomAdversary (org.neo4j.adversaries.RandomAdversary)5 AdversarialFileSystemAbstraction (org.neo4j.adversaries.fs.AdversarialFileSystemAbstraction)5 SingleFilePageSwapperFactory (org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory)4 InputStream (java.io.InputStream)3 FileLock (java.nio.channels.FileLock)3 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)3 IOException (java.io.IOException)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)2 StoreFileChannel (org.neo4j.io.fs.StoreFileChannel)2 PageCache (org.neo4j.io.pagecache.PageCache)2 PagedFile (org.neo4j.io.pagecache.PagedFile)2