Examples with GridConcurrentMultiPairQueue org.apache.ignite.internal.util.GridConcurrentMultiPairQueue used on opensource projects

Example 1 with GridConcurrentMultiPairQueue

use of org.apache.ignite.internal.util.GridConcurrentMultiPairQueue in project ignite by apache.

the class CheckpointWorkflow method splitAndSortCpPagesIfNeeded.

/**
 * Reorders list of checkpoint pages and splits them into appropriate number of sublists according to {@link
 * DataStorageConfiguration#getCheckpointThreads()} and {@link DataStorageConfiguration#getCheckpointWriteOrder()}.
 *
 * @param cpPages Checkpoint pages with overall count and user pages info.
 */
private GridConcurrentMultiPairQueue<PageMemoryEx, FullPageId> splitAndSortCpPagesIfNeeded(CheckpointPagesInfoHolder cpPages) throws IgniteCheckedException {
    Set<T2<PageMemoryEx, FullPageId[]>> cpPagesPerRegion = new HashSet<>();
    int realPagesArrSize = 0;
    int totalPagesCnt = cpPages.pagesNum();
    for (Map.Entry<PageMemoryEx, GridMultiCollectionWrapper<FullPageId>> regPages : cpPages.cpPages()) {
        FullPageId[] pages = new FullPageId[regPages.getValue().size()];
        int pagePos = 0;
        for (int i = 0; i < regPages.getValue().collectionsSize(); i++) {
            for (FullPageId page : regPages.getValue().innerCollection(i)) {
                if (realPagesArrSize++ == totalPagesCnt)
                    throw new AssertionError("Incorrect estimated dirty pages number: " + totalPagesCnt);
                pages[pagePos++] = page;
            }
        }
        // Some pages may have been already replaced.
        if (pagePos != pages.length)
            cpPagesPerRegion.add(new T2<>(regPages.getKey(), Arrays.copyOf(pages, pagePos)));
        else
            cpPagesPerRegion.add(new T2<>(regPages.getKey(), pages));
    }
    if (checkpointWriteOrder == CheckpointWriteOrder.SEQUENTIAL) {
        Comparator<FullPageId> cmp = Comparator.comparingInt(FullPageId::groupId).thenComparingLong(FullPageId::effectivePageId);
        ForkJoinPool pool = null;
        for (T2<PageMemoryEx, FullPageId[]> pagesPerReg : cpPagesPerRegion) {
            if (pagesPerReg.getValue().length >= parallelSortThreshold)
                pool = parallelSortInIsolatedPool(pagesPerReg.get2(), cmp, pool);
            else
                Arrays.sort(pagesPerReg.get2(), cmp);
        }
        if (pool != null)
            pool.shutdown();
    }
    return new GridConcurrentMultiPairQueue<>(cpPagesPerRegion);
}

Example 2 with GridConcurrentMultiPairQueue

use of org.apache.ignite.internal.util.GridConcurrentMultiPairQueue in project ignite by apache.

the class CheckpointPagesWriter method writePages.

/**
 * @param writePageIds Collections of pages to write.
 * @return pagesToRetry Pages which should be retried.
 */
private GridConcurrentMultiPairQueue<PageMemoryEx, FullPageId> writePages(GridConcurrentMultiPairQueue<PageMemoryEx, FullPageId> writePageIds) throws IgniteCheckedException {
    Map<PageMemoryEx, List<FullPageId>> pagesToRetry = new HashMap<>();
    CheckpointMetricsTracker tracker = persStoreMetrics.metricsEnabled() ? this.tracker : null;
    Map<PageMemoryEx, PageStoreWriter> pageStoreWriters = new HashMap<>();
    ByteBuffer tmpWriteBuf = threadBuf.get();
    boolean throttlingEnabled = throttlingPolicy != PageMemoryImpl.ThrottlingPolicy.DISABLED;
    GridConcurrentMultiPairQueue.Result<PageMemoryEx, FullPageId> res = new GridConcurrentMultiPairQueue.Result<>();
    while (writePageIds.next(res)) {
        if (shutdownNow.getAsBoolean())
            break;
        beforePageWrite.run();
        FullPageId fullId = res.getValue();
        PageMemoryEx pageMem = res.getKey();
        snapshotMgr.beforePageWrite(fullId);
        tmpWriteBuf.rewind();
        PageStoreWriter pageStoreWriter = pageStoreWriters.computeIfAbsent(pageMem, pageMemEx -> createPageStoreWriter(pageMemEx, pagesToRetry));
        pageMem.checkpointWritePage(fullId, tmpWriteBuf, pageStoreWriter, tracker);
        if (throttlingEnabled) {
            while (pageMem.isCpBufferOverflowThresholdExceeded()) {
                FullPageId cpPageId = pageMem.pullPageFromCpBuffer();
                if (cpPageId.equals(FullPageId.NULL_PAGE))
                    break;
                snapshotMgr.beforePageWrite(cpPageId);
                tmpWriteBuf.rewind();
                pageMem.checkpointWritePage(cpPageId, tmpWriteBuf, pageStoreWriter, tracker);
            }
        }
    }
    return pagesToRetry.isEmpty() ? GridConcurrentMultiPairQueue.EMPTY : new GridConcurrentMultiPairQueue<>(pagesToRetry);
}