Examples with GridMultiCollectionWrapper org.apache.ignite.internal.util.GridMultiCollectionWrapper used on opensource projects

Example 1 with GridMultiCollectionWrapper

use of org.apache.ignite.internal.util.GridMultiCollectionWrapper in project ignite by apache.

the class PageMemoryImpl method beginCheckpoint.

/**
 * {@inheritDoc}
 */
@Override
public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException {
    if (segments == null)
        return new GridMultiCollectionWrapper<>(Collections.<FullPageId>emptyList());
    Collection[] collections = new Collection[segments.length];
    for (int i = 0; i < segments.length; i++) {
        Segment seg = segments[i];
        if (seg.segCheckpointPages != null)
            throw new IgniteException("Failed to begin checkpoint (it is already in progress).");
        collections[i] = seg.segCheckpointPages = seg.dirtyPages;
        seg.dirtyPages = new GridConcurrentHashSet<>();
    }
    memMetrics.resetDirtyPages();
    if (throttlingPlc != ThrottlingPolicy.DISABLED)
        writeThrottle.onBeginCheckpoint();
    return new GridMultiCollectionWrapper<>(collections);
}

Example 2 with GridMultiCollectionWrapper

use of org.apache.ignite.internal.util.GridMultiCollectionWrapper in project ignite by apache.

the class CheckpointWorkflow method splitAndSortCpPagesIfNeeded.

/**
 * Reorders list of checkpoint pages and splits them into appropriate number of sublists according to {@link
 * DataStorageConfiguration#getCheckpointThreads()} and {@link DataStorageConfiguration#getCheckpointWriteOrder()}.
 *
 * @param cpPages Checkpoint pages with overall count and user pages info.
 */
private GridConcurrentMultiPairQueue<PageMemoryEx, FullPageId> splitAndSortCpPagesIfNeeded(CheckpointPagesInfoHolder cpPages) throws IgniteCheckedException {
    Set<T2<PageMemoryEx, FullPageId[]>> cpPagesPerRegion = new HashSet<>();
    int realPagesArrSize = 0;
    int totalPagesCnt = cpPages.pagesNum();
    for (Map.Entry<PageMemoryEx, GridMultiCollectionWrapper<FullPageId>> regPages : cpPages.cpPages()) {
        FullPageId[] pages = new FullPageId[regPages.getValue().size()];
        int pagePos = 0;
        for (int i = 0; i < regPages.getValue().collectionsSize(); i++) {
            for (FullPageId page : regPages.getValue().innerCollection(i)) {
                if (realPagesArrSize++ == totalPagesCnt)
                    throw new AssertionError("Incorrect estimated dirty pages number: " + totalPagesCnt);
                pages[pagePos++] = page;
            }
        }
        // Some pages may have been already replaced.
        if (pagePos != pages.length)
            cpPagesPerRegion.add(new T2<>(regPages.getKey(), Arrays.copyOf(pages, pagePos)));
        else
            cpPagesPerRegion.add(new T2<>(regPages.getKey(), pages));
    }
    if (checkpointWriteOrder == CheckpointWriteOrder.SEQUENTIAL) {
        Comparator<FullPageId> cmp = Comparator.comparingInt(FullPageId::groupId).thenComparingLong(FullPageId::effectivePageId);
        ForkJoinPool pool = null;
        for (T2<PageMemoryEx, FullPageId[]> pagesPerReg : cpPagesPerRegion) {
            if (pagesPerReg.getValue().length >= parallelSortThreshold)
                pool = parallelSortInIsolatedPool(pagesPerReg.get2(), cmp, pool);
            else
                Arrays.sort(pagesPerReg.get2(), cmp);
        }
        if (pool != null)
            pool.shutdown();
    }
    return new GridConcurrentMultiPairQueue<>(cpPagesPerRegion);
}

Example 3 with GridMultiCollectionWrapper

use of org.apache.ignite.internal.util.GridMultiCollectionWrapper in project ignite by apache.

the class PageMemoryImplTest method runThrottlingEmptifyCpBufFirst.

/**
 * @throws Exception if failed.
 */
public void runThrottlingEmptifyCpBufFirst(PageMemoryImpl.ThrottlingPolicy plc) throws Exception {
    TestPageStoreManager pageStoreMgr = new TestPageStoreManager();
    final List<FullPageId> allocated = new ArrayList<>();
    int pagesForStartThrottling = 10;
    // Number of pages which were poll from checkpoint buffer for throttling.
    AtomicInteger cpBufferPollPages = new AtomicInteger();
    // Create a 1 mb page memory.
    PageMemoryImpl memory = createPageMemory(1, plc, pageStoreMgr, pageStoreMgr, (IgniteInClosure<FullPageId>) fullPageId -> {
        assertEquals(cpBufferPollPages.incrementAndGet(), pageStoreMgr.storedPages.size());
    });
    assert pagesForStartThrottling < memory.checkpointBufferPagesSize() / 3;
    for (int i = 0; i < pagesForStartThrottling + (memory.checkpointBufferPagesSize() * 2 / 3); i++) {
        long id = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
        FullPageId fullId = new FullPageId(id, 1);
        allocated.add(fullId);
        writePage(memory, fullId, (byte) 1);
    }
    GridMultiCollectionWrapper<FullPageId> markedPages = memory.beginCheckpoint(new GridFinishedFuture());
    for (int i = 0; i < pagesForStartThrottling + (memory.checkpointBufferPagesSize() * 2 / 3); i++) writePage(memory, allocated.get(i), (byte) 1);
    doCheckpoint(markedPages, memory, pageStoreMgr);
    // There is 'pagesForStartThrottling - 1' because we should write pagesForStartThrottling pages
    // from checkpoint buffer before throttling will be disabled but at least one page always would be written
    // outside of throttling and in our case we certainly know that this page is also contained in checkpoint buffer
    // (because all of our pages are in checkpoint buffer).
    assertEquals(pagesForStartThrottling - 1, cpBufferPollPages.get());
}

Example 4 with GridMultiCollectionWrapper

use of org.apache.ignite.internal.util.GridMultiCollectionWrapper in project ignite by apache.

the class PageMemoryImpl method beginCheckpoint.

/**
 * {@inheritDoc}
 */
@Override
public GridMultiCollectionWrapper<FullPageId> beginCheckpoint(IgniteInternalFuture allowToReplace) throws IgniteException {
    if (segments == null)
        return new GridMultiCollectionWrapper<>(Collections.emptyList());
    Collection[] collections = new Collection[segments.length];
    for (int i = 0; i < segments.length; i++) {
        Segment seg = segments[i];
        if (seg.checkpointPages != null)
            throw new IgniteException("Failed to begin checkpoint (it is already in progress).");
        Collection<FullPageId> dirtyPages = seg.dirtyPages;
        collections[i] = dirtyPages;
        seg.checkpointPages = new CheckpointPages(dirtyPages, allowToReplace);
        seg.resetDirtyPages();
    }
    safeToUpdate.set(true);
    dataRegionMetrics.resetDirtyPages();
    if (throttlingPlc != ThrottlingPolicy.DISABLED)
        writeThrottle.onBeginCheckpoint();
    return new GridMultiCollectionWrapper<>(collections);
}

Example 5 with GridMultiCollectionWrapper

use of org.apache.ignite.internal.util.GridMultiCollectionWrapper in project ignite by apache.

the class PageMemoryImplTest method doCheckpoint.

/**
 * @param cpPages Checkpoint pages acuiqred by {@code beginCheckpoint()}.
 * @param memory Page memory.
 * @param pageStoreMgr Test page store manager.
 * @throws Exception If failed.
 */
private void doCheckpoint(GridMultiCollectionWrapper<FullPageId> cpPages, PageMemoryImpl memory, TestPageStoreManager pageStoreMgr) throws Exception {
    PageStoreWriter pageStoreWriter = (fullPageId, buf, tag) -> {
        assertNotNull(tag);
        pageStoreMgr.write(fullPageId.groupId(), fullPageId.pageId(), buf, 1, false);
    };
    for (FullPageId cpPage : cpPages) {
        byte[] data = new byte[PAGE_SIZE];
        ByteBuffer buf = ByteBuffer.wrap(data);
        memory.checkpointWritePage(cpPage, buf, pageStoreWriter, null);
        while (memory.isCpBufferOverflowThresholdExceeded()) {
            FullPageId cpPageId = memory.pullPageFromCpBuffer();
            if (cpPageId.equals(FullPageId.NULL_PAGE))
                break;
            ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(memory.pageSize());
            tmpWriteBuf.order(ByteOrder.nativeOrder());
            tmpWriteBuf.rewind();
            memory.checkpointWritePage(cpPageId, tmpWriteBuf, pageStoreWriter, null);
        }
    }
    memory.finishCheckpoint();
}