Search in sources :

Example 1 with Futures

use of org.neo4j.util.concurrent.Futures in project neo4j by neo4j.

the class PageSwapperTest method concurrentPositionedVectoredReadsAndWritesMustNotInterfere.

@Test
void concurrentPositionedVectoredReadsAndWritesMustNotInterfere() throws Exception {
    Path file = file("file");
    PageSwapperFactory factory = createSwapperFactory(getFs());
    final PageSwapper swapper = createSwapperAndFile(factory, file, 4);
    final int pageCount = 100;
    final int iterations = 20000;
    final CountDownLatch startLatch = new CountDownLatch(1);
    long output = createPage(4);
    for (int i = 0; i < pageCount; i++) {
        putInt(output, 0, i + 1);
        write(swapper, i, output);
    }
    Callable<Void> work = () -> {
        ThreadLocalRandom rng = ThreadLocalRandom.current();
        int length = 10;
        int pageSize = 4;
        long[] pages = new long[length];
        int[] sizes = new int[length];
        for (int i = 0; i < length; i++) {
            pages[i] = createPage(pageSize);
            sizes[i] = pageSize;
        }
        startLatch.await();
        for (int i = 0; i < iterations; i++) {
            long startFilePageId = rng.nextLong(0, pageCount - pages.length);
            if (rng.nextBoolean()) {
                // Do read
                long bytesRead = read(swapper, startFilePageId, pages, sizes, pages.length);
                assertThat(bytesRead).isEqualTo(pages.length * 4L);
                for (int j = 0; j < pages.length; j++) {
                    int expectedValue = (int) (1 + j + startFilePageId);
                    int actualValue = getInt(pages[j], 0);
                    assertThat(actualValue).isEqualTo(expectedValue);
                }
            } else {
                // Do write
                for (int j = 0; j < pages.length; j++) {
                    int value = (int) (1 + j + startFilePageId);
                    putInt(pages[j], 0, value);
                }
                assertThat(write(swapper, startFilePageId, pages, sizes, pages.length, pages.length)).isEqualTo(pages.length * 4L);
            }
        }
        return null;
    };
    int threads = 8;
    ExecutorService executor = null;
    try {
        executor = Executors.newFixedThreadPool(threads, r -> {
            Thread thread = Executors.defaultThreadFactory().newThread(r);
            thread.setDaemon(true);
            return thread;
        });
        List<Future<?>> futures = new ArrayList<>(threads);
        for (int i = 0; i < threads; i++) {
            futures.add(executor.submit(work));
        }
        startLatch.countDown();
        Futures.getAll(futures);
    } finally {
        if (executor != null) {
            executor.shutdown();
        }
    }
}
Also used : Path(java.nio.file.Path) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) BeforeEach(org.junit.jupiter.api.BeforeEach) NoSuchFileException(java.nio.file.NoSuchFileException) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) LocalMemoryTracker(org.neo4j.memory.LocalMemoryTracker) DISABLED(org.neo4j.io.pagecache.IOController.DISABLED) DisabledOnOs(org.junit.jupiter.api.condition.DisabledOnOs) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) Inject(org.neo4j.test.extension.Inject) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) EnabledOnOs(org.junit.jupiter.api.condition.EnabledOnOs) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) TestDirectoryExtension(org.neo4j.test.extension.testdirectory.TestDirectoryExtension) Path(java.nio.file.Path) ExecutorService(java.util.concurrent.ExecutorService) SwapperSet(org.neo4j.io.pagecache.impl.muninn.SwapperSet) Futures(org.neo4j.util.concurrent.Futures) OS(org.junit.jupiter.api.condition.OS) ClosedChannelException(java.nio.channels.ClosedChannelException) MemoryAllocator(org.neo4j.io.mem.MemoryAllocator) TestDirectory(org.neo4j.test.rule.TestDirectory) IOException(java.io.IOException) Executors(java.util.concurrent.Executors) Test(org.junit.jupiter.api.Test) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicLong(java.util.concurrent.atomic.AtomicLong) AfterEach(org.junit.jupiter.api.AfterEach) List(java.util.List) UnsafeUtil(org.neo4j.internal.unsafe.UnsafeUtil) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) KibiByte(org.neo4j.io.ByteUnit.KibiByte) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) ArrayList(java.util.ArrayList) CountDownLatch(java.util.concurrent.CountDownLatch) ExecutorService(java.util.concurrent.ExecutorService) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Future(java.util.concurrent.Future) Test(org.junit.jupiter.api.Test)

Example 2 with Futures

use of org.neo4j.util.concurrent.Futures in project neo4j by neo4j.

the class TransactionLogFileTest method shouldBatchUpMultipleWaitingForceRequests.

@Test
void shouldBatchUpMultipleWaitingForceRequests() throws Throwable {
    LogFiles logFiles = buildLogFiles();
    life.start();
    life.add(logFiles);
    LogFile logFile = logFiles.getLogFile();
    var capturingChannel = wrappingFileSystem.getCapturingChannel();
    var flushesBefore = capturingChannel.getFlushCounter().get();
    var writesBefore = capturingChannel.getWriteAllCounter().get();
    ReentrantLock writeAllLock = capturingChannel.getWriteAllLock();
    writeAllLock.lock();
    int executors = 10;
    var executorService = Executors.newFixedThreadPool(executors);
    try {
        List<Future<?>> futures = Stream.iterate(0, i -> i + 1).limit(executors).map(v -> executorService.submit(() -> logFile.forceAfterAppend(LogAppendEvent.NULL))).collect(toList());
        while (!writeAllLock.hasQueuedThreads()) {
            parkNanos(100);
        }
        writeAllLock.unlock();
        assertThat(futures).hasSize(executors);
        Futures.getAll(futures);
    } finally {
        if (writeAllLock.isLocked()) {
            writeAllLock.unlock();
        }
        executorService.shutdownNow();
    }
    assertThat(capturingChannel.getFlushCounter().get() - flushesBefore).isLessThanOrEqualTo(executors);
    assertThat(capturingChannel.getWriteAllCounter().get() - writesBefore).isLessThanOrEqualTo(executors);
}
Also used : LogFile(org.neo4j.kernel.impl.transaction.log.files.LogFile) ReentrantLock(java.util.concurrent.locks.ReentrantLock) BeforeEach(org.junit.jupiter.api.BeforeEach) LogHeaderReader.readLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderReader.readLogHeader) NativeCallResult(org.neo4j.internal.nativeimpl.NativeCallResult) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) LogFilesBuilder(org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) ByteBuffer(java.nio.ByteBuffer) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) Mockito.doThrow(org.mockito.Mockito.doThrow) Future(java.util.concurrent.Future) StoreChannel(org.neo4j.io.fs.StoreChannel) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) LogVersionRepository(org.neo4j.storageengine.api.LogVersionRepository) NULL(org.neo4j.io.pagecache.context.CursorContext.NULL) Path(java.nio.file.Path) Futures(org.neo4j.util.concurrent.Futures) TransactionLogFilesHelper(org.neo4j.kernel.impl.transaction.log.files.TransactionLogFilesHelper) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) Executors(java.util.concurrent.Executors) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) Test(org.junit.jupiter.api.Test) List(java.util.List) INSTANCE(org.neo4j.memory.EmptyMemoryTracker.INSTANCE) Stream(java.util.stream.Stream) TestLogEntryReader.logEntryReader(org.neo4j.kernel.impl.transaction.log.TestLogEntryReader.logEntryReader) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) DelegatingStoreChannel(org.neo4j.io.fs.DelegatingStoreChannel) Mockito.mock(org.mockito.Mockito.mock) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) LogFiles(org.neo4j.kernel.impl.transaction.log.files.LogFiles) DelegatingFileSystemAbstraction(org.neo4j.io.fs.DelegatingFileSystemAbstraction) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) LockSupport.parkNanos(java.util.concurrent.locks.LockSupport.parkNanos) LogFile(org.neo4j.kernel.impl.transaction.log.files.LogFile) Inject(org.neo4j.test.extension.Inject) StoreId(org.neo4j.storageengine.api.StoreId) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ByteUnit(org.neo4j.io.ByteUnit) LifeExtension(org.neo4j.test.extension.LifeExtension) ReentrantLock(java.util.concurrent.locks.ReentrantLock) NativeAccess(org.neo4j.internal.nativeimpl.NativeAccess) IOException(java.io.IOException) LogAppendEvent(org.neo4j.kernel.impl.transaction.tracing.LogAppendEvent) Mockito.when(org.mockito.Mockito.when) SimpleLogVersionRepository(org.neo4j.kernel.impl.transaction.SimpleLogVersionRepository) Mockito.verify(org.mockito.Mockito.verify) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) IncompleteLogHeaderException(org.neo4j.kernel.impl.transaction.log.entry.IncompleteLogHeaderException) Collectors.toList(java.util.stream.Collectors.toList) CURRENT_FORMAT_LOG_HEADER_SIZE(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_FORMAT_LOG_HEADER_SIZE) ReadableChannel(org.neo4j.io.fs.ReadableChannel) TransactionIdStore(org.neo4j.storageengine.api.TransactionIdStore) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) SimpleTransactionIdStore(org.neo4j.kernel.impl.transaction.SimpleTransactionIdStore) LogFiles(org.neo4j.kernel.impl.transaction.log.files.LogFiles) Future(java.util.concurrent.Future) Test(org.junit.jupiter.api.Test)

Aggregations

IOException (java.io.IOException)2 Path (java.nio.file.Path)2 List (java.util.List)2 Executors (java.util.concurrent.Executors)2 Future (java.util.concurrent.Future)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)2 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)2 Assertions.assertThrows (org.junit.jupiter.api.Assertions.assertThrows)2 Assertions.assertTrue (org.junit.jupiter.api.Assertions.assertTrue)2 BeforeEach (org.junit.jupiter.api.BeforeEach)2 Test (org.junit.jupiter.api.Test)2 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)2 ByteBuffer (java.nio.ByteBuffer)1 ClosedChannelException (java.nio.channels.ClosedChannelException)1 NoSuchFileException (java.nio.file.NoSuchFileException)1 ArrayList (java.util.ArrayList)1 Callable (java.util.concurrent.Callable)1 ConcurrentLinkedQueue (java.util.concurrent.ConcurrentLinkedQueue)1 CountDownLatch (java.util.concurrent.CountDownLatch)1