Examples with PagedFile org.neo4j.io.pagecache.PagedFile used on opensource projects

Example 11 with PagedFile

use of org.neo4j.io.pagecache.PagedFile in project neo4j by neo4j.

the class PagedByteChannelsTest method mustCloseCursorOnClose.

@Theory
public void mustCloseCursorOnClose(ThrowingFunction<PagedFile, ? extends Channel, IOException> channelConstructor) throws Exception {
    AtomicInteger closeCounter = new AtomicInteger();
    PagedFile pf = new StubPagedFile(8192) {

        @Override
        public PageCursor io(long pageId, int pf_flags) throws IOException {
            return new DelegatingPageCursor(super.io(pageId, pf_flags)) {

                @Override
                public void close() {
                    super.close();
                    closeCounter.getAndIncrement();
                }
            };
        }
    };
    channelConstructor.apply(pf).close();
    assertThat(closeCounter.get(), is(1));
}

Example 12 with PagedFile

use of org.neo4j.io.pagecache.PagedFile in project neo4j by neo4j.

the class MetaDataStoreTest method transactionCommittedMustBeAtomic.

@Test
public void transactionCommittedMustBeAtomic() throws Throwable {
    try (MetaDataStore store = newMetaDataStore()) {
        PagedFile pf = store.storeFile;
        store.transactionCommitted(2, 2, 2);
        AtomicLong writeCount = new AtomicLong();
        AtomicLong fileReadCount = new AtomicLong();
        AtomicLong apiReadCount = new AtomicLong();
        int upperLimit = 10_000;
        int lowerLimit = 100;
        long endTime = currentTimeMillis() + SECONDS.toMillis(10);
        Race race = new Race();
        race.withEndCondition(() -> writeCount.get() >= upperLimit && fileReadCount.get() >= upperLimit && apiReadCount.get() >= upperLimit);
        race.withEndCondition(() -> writeCount.get() >= lowerLimit && fileReadCount.get() >= lowerLimit && apiReadCount.get() >= lowerLimit && currentTimeMillis() >= endTime);
        race.addContestants(3, () -> {
            long count = writeCount.incrementAndGet();
            store.transactionCommitted(count, count, count);
        });
        race.addContestants(3, throwing(() -> {
            try (PageCursor cursor = pf.io(0, PagedFile.PF_SHARED_READ_LOCK)) {
                assertTrue(cursor.next());
                long id, checksum;
                do {
                    id = store.getRecordValue(cursor, MetaDataStore.Position.LAST_TRANSACTION_ID);
                    checksum = store.getRecordValue(cursor, MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM);
                } while (cursor.shouldRetry());
                assertIdEqualsChecksum(id, checksum, "file");
                fileReadCount.incrementAndGet();
            }
        }));
        race.addContestants(3, () -> {
            TransactionId transaction = store.getLastCommittedTransaction();
            assertIdEqualsChecksum(transaction.transactionId(), transaction.checksum(), "API");
            apiReadCount.incrementAndGet();
        });
        race.go();
    }
}

Example 13 with PagedFile

use of org.neo4j.io.pagecache.PagedFile in project neo4j by neo4j.

the class SwitchToSlaveCopyThenBranchTest method shouldNotBranchStoreUnlessWeHaveCopiedDownAReplacement.

@Test
public void shouldNotBranchStoreUnlessWeHaveCopiedDownAReplacement() throws Throwable {
    // Given
    StoreCopyClient storeCopyClient = mock(StoreCopyClient.class);
    doAnswer(invocation -> {
        MoveAfterCopy moveAfterCopy = invocation.getArgumentAt(2, MoveAfterCopy.class);
        moveAfterCopy.move(Stream.empty(), new File(""), new File(""));
        return null;
    }).when(storeCopyClient).copyStore(any(StoreCopyClient.StoreCopyRequester.class), any(CancellationRequest.class), any(MoveAfterCopy.class));
    PageCache pageCacheMock = mock(PageCache.class);
    PagedFile pagedFileMock = mock(PagedFile.class);
    when(pagedFileMock.getLastPageId()).thenReturn(1L);
    when(pageCacheMock.map(any(File.class), anyInt())).thenReturn(pagedFileMock);
    SwitchToSlaveCopyThenBranch switchToSlave = newSwitchToSlaveSpy(pageCacheMock, storeCopyClient);
    URI masterUri = new URI("cluster://localhost?serverId=1");
    URI me = new URI("cluster://localhost?serverId=2");
    CancellationRequest cancellationRequest = CancellationRequest.NEVER_CANCELLED;
    MasterClient masterClient = mock(MasterClient.class);
    when(masterClient.handshake(anyLong(), any(StoreId.class))).thenThrow(new BranchedDataException(""));
    TransactionIdStore transactionIdStore = mock(TransactionIdStore.class);
    when(transactionIdStore.getLastCommittedTransaction()).thenReturn(new TransactionId(42, 42, 42));
    when(transactionIdStore.getLastCommittedTransactionId()).thenReturn(TransactionIdStore.BASE_TX_ID);
    // When
    BranchedDataPolicy branchPolicy = mock(BranchedDataPolicy.class);
    switchToSlave.stopServicesAndHandleBranchedStore(branchPolicy, masterUri, me, cancellationRequest);
    // Then
    InOrder inOrder = Mockito.inOrder(storeCopyClient, branchPolicy);
    inOrder.verify(storeCopyClient).copyStore(any(StoreCopyClient.StoreCopyRequester.class), any(CancellationRequest.class), any(MoveAfterCopy.class));
    inOrder.verify(branchPolicy).handle(new File(""), pageCacheMock, NullLogService.getInstance());
}

Example 14 with PagedFile

use of org.neo4j.io.pagecache.PagedFile in project neo4j by neo4j.

the class StreamToDisk method write.

@Override
public void write(String destination, int requiredAlignment, byte[] data) throws IOException {
    File fileName = new File(storeDir, destination);
    fs.mkdirs(fileName.getParentFile());
    fileCopyMonitor.copyFile(fileName);
    if (StoreType.shouldBeManagedByPageCache(destination)) {
        WritableByteChannel channel = channels.get(destination);
        if (channel == null) {
            int filePageSize = pageCache.pageSize() - pageCache.pageSize() % requiredAlignment;
            PagedFile pagedFile = pageCache.map(fileName, filePageSize, StandardOpenOption.CREATE);
            channel = pagedFile.openWritableByteChannel();
            pagedFiles.put(destination, pagedFile);
            channels.put(destination, channel);
        }
        ByteBuffer buffer = ByteBuffer.wrap(data);
        while (buffer.hasRemaining()) {
            channel.write(buffer);
        }
    } else {
        try (OutputStream outputStream = fs.openAsOutputStream(fileName, true)) {
            outputStream.write(data);
        }
    }
}

Example 15 with PagedFile

use of org.neo4j.io.pagecache.PagedFile in project neo4j by neo4j.

the class GetStoreRequestHandler method channelRead0.

@Override
protected void channelRead0(ChannelHandlerContext ctx, GetStoreRequest msg) throws Exception {
    if (!msg.expectedStoreId().equalToKernelStoreId(dataSource.get().getStoreId())) {
        endStoreCopy(SUCCESS, ctx, -1);
    } else {
        CheckPointer checkPointer = checkPointerSupplier.get();
        long lastCheckPointedTx;
        try (Resource lock = mutex.storeCopy(() -> checkPointer.tryCheckPoint(new SimpleTriggerInfo("Store copy")));
            ResourceIterator<StoreFileMetadata> files = dataSource.get().listStoreFiles(false)) {
            lastCheckPointedTx = checkPointer.lastCheckPointedTransactionId();
            while (files.hasNext()) {
                StoreFileMetadata fileMetadata = files.next();
                File file = fileMetadata.file();
                log.debug("Sending file " + file);
                ctx.writeAndFlush(ResponseMessageType.FILE);
                ctx.writeAndFlush(new FileHeader(relativePath(dataSource.get().getStoreDir(), file), fileMetadata.recordSize()));
                Optional<PagedFile> existingMapping = pageCache.getExistingMapping(file);
                if (existingMapping.isPresent()) {
                    try (PagedFile pagedFile = existingMapping.get()) {
                        ctx.writeAndFlush(new FileSender(pagedFile.openReadableByteChannel()));
                    }
                } else {
                    ctx.writeAndFlush(new FileSender(fs.open(file, "r")));
                }
            }
        }
        endStoreCopy(SUCCESS, ctx, lastCheckPointedTx);
    }
    protocol.expect(State.MESSAGE_TYPE);
}