Examples with GroupPartitionId org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId used on opensource projects

Example 11 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class SnapshotFutureTask method onCheckpointBegin.

/**
 * {@inheritDoc}
 */
@Override
public void onCheckpointBegin(Context ctx) {
    if (stopping())
        return;
    assert !processed.isEmpty() : "Partitions to process must be collected under checkpoint mark phase";
    wrapExceptionIfStarted(() -> snpSndr.init(processed.values().stream().mapToInt(Set::size).sum())).run();
    // there is no error happen on task init.
    if (!startedFut.onDone())
        return;
    // Submit all tasks for partitions and deltas processing.
    List<CompletableFuture<Void>> futs = new ArrayList<>();
    if (log.isInfoEnabled()) {
        log.info("Submit partition processing tasks to the snapshot execution pool " + "[map=" + compactGroupPartitions(partFileLengths.keySet()) + ", totalSize=" + U.humanReadableByteCount(partFileLengths.values().stream().mapToLong(v -> v).sum()) + ']');
    }
    Collection<BinaryType> binTypesCopy = cctx.kernalContext().cacheObjects().metadata(Collections.emptyList()).values();
    // Process binary meta.
    futs.add(CompletableFuture.runAsync(wrapExceptionIfStarted(() -> snpSndr.sendBinaryMeta(binTypesCopy)), snpSndr.executor()));
    List<Map<Integer, MappedName>> mappingsCopy = cctx.kernalContext().marshallerContext().getCachedMappings();
    // Process marshaller meta.
    futs.add(CompletableFuture.runAsync(wrapExceptionIfStarted(() -> snpSndr.sendMarshallerMeta(mappingsCopy)), snpSndr.executor()));
    // Send configuration files of all cache groups.
    for (CacheConfigurationSender ccfgSndr : ccfgSndrs) futs.add(CompletableFuture.runAsync(wrapExceptionIfStarted(ccfgSndr::sendCacheConfig), snpSndr.executor()));
    try {
        for (Map.Entry<Integer, Set<Integer>> e : processed.entrySet()) {
            int grpId = e.getKey();
            String cacheDirName = pageStore.cacheDirName(grpId);
            // Process partitions for a particular cache group.
            for (int partId : e.getValue()) {
                GroupPartitionId pair = new GroupPartitionId(grpId, partId);
                Long partLen = partFileLengths.get(pair);
                CompletableFuture<Void> fut0 = CompletableFuture.runAsync(wrapExceptionIfStarted(() -> {
                    snpSndr.sendPart(getPartitionFile(pageStore.workDir(), cacheDirName, partId), cacheDirName, pair, partLen);
                    // Stop partition writer.
                    partDeltaWriters.get(pair).markPartitionProcessed();
                }), snpSndr.executor()).runAfterBothAsync(cpEndFut, wrapExceptionIfStarted(() -> {
                    File delta = partDeltaWriters.get(pair).deltaFile;
                    try {
                        // Atomically creates a new, empty delta file if and only if
                        // a file with this name does not yet exist.
                        delta.createNewFile();
                    } catch (IOException ex) {
                        throw new IgniteCheckedException(ex);
                    }
                    snpSndr.sendDelta(delta, cacheDirName, pair);
                    boolean deleted = delta.delete();
                    assert deleted;
                }), snpSndr.executor());
                futs.add(fut0);
            }
        }
        int futsSize = futs.size();
        CompletableFuture.allOf(futs.toArray(new CompletableFuture[futsSize])).whenComplete((res, t) -> {
            assert t == null : "Exception must never be thrown since a wrapper is used " + "for each snapshot task: " + t;
            closeAsync();
        });
    } catch (IgniteCheckedException e) {
        acceptException(e);
    }
}

Example 12 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class SnapshotResponseRemoteFutureTask method start.

/**
 * {@inheritDoc}
 */
@Override
public boolean start() {
    if (F.isEmpty(parts))
        return false;
    try {
        List<GroupPartitionId> handled = new ArrayList<>();
        for (Map.Entry<Integer, Set<Integer>> e : parts.entrySet()) {
            ofNullable(e.getValue()).orElse(Collections.emptySet()).forEach(p -> handled.add(new GroupPartitionId(e.getKey(), p)));
        }
        snpSndr.init(handled.size());
        File snpDir = cctx.snapshotMgr().snapshotLocalDir(snpName);
        List<CompletableFuture<Void>> futs = new ArrayList<>();
        List<SnapshotMetadata> metas = cctx.snapshotMgr().readSnapshotMetadatas(snpName);
        for (SnapshotMetadata meta : metas) {
            Map<Integer, Set<Integer>> parts0 = meta.partitions();
            if (F.isEmpty(parts0))
                continue;
            handled.removeIf(gp -> {
                if (ofNullable(parts0.get(gp.getGroupId())).orElse(Collections.emptySet()).contains(gp.getPartitionId())) {
                    futs.add(CompletableFuture.runAsync(() -> {
                        if (err.get() != null)
                            return;
                        File cacheDir = cacheDirectory(new File(snpDir, databaseRelativePath(meta.folderName())), gp.getGroupId());
                        if (cacheDir == null) {
                            throw new IgniteException("Cache directory not found [snpName=" + snpName + ", meta=" + meta + ", pair=" + gp + ']');
                        }
                        File snpPart = getPartitionFile(cacheDir.getParentFile(), cacheDir.getName(), gp.getPartitionId());
                        if (!snpPart.exists()) {
                            throw new IgniteException("Snapshot partition file not found [cacheDir=" + cacheDir + ", pair=" + gp + ']');
                        }
                        snpSndr.sendPart(snpPart, cacheDir.getName(), gp, snpPart.length());
                    }, snpSndr.executor()).whenComplete((r, t) -> err.compareAndSet(null, t)));
                    return true;
                }
                return false;
            });
        }
        if (!handled.isEmpty()) {
            err.compareAndSet(null, new IgniteException("Snapshot partitions missed on local node [snpName=" + snpName + ", missed=" + handled + ']'));
        }
        int size = futs.size();
        CompletableFuture.allOf(futs.toArray(new CompletableFuture[size])).whenComplete((r, t) -> {
            Throwable th = ofNullable(err.get()).orElse(t);
            if (th == null && log.isInfoEnabled()) {
                log.info("Snapshot partitions have been sent to the remote node [snpName=" + snpName + ", rmtNodeId=" + srcNodeId + ']');
            }
            close(th);
        });
        return true;
    } catch (Throwable t) {
        if (err.compareAndSet(null, t))
            close(t);
        return false;
    }
}

Example 13 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class GridCacheDatabaseSharedManager method reserveHistoryForPreloading.

/**
 * {@inheritDoc}
 */
@Override
public boolean reserveHistoryForPreloading(Map<T2<Integer, Integer>, Long> reservationMap) {
    Map<GroupPartitionId, CheckpointEntry> entries = checkpointHistory().searchCheckpointEntry(reservationMap);
    if (F.isEmpty(entries))
        return false;
    WALPointer oldestWALPointerToReserve = null;
    for (CheckpointEntry cpE : entries.values()) {
        WALPointer ptr = cpE.checkpointMark();
        if (ptr == null)
            return false;
        if (oldestWALPointerToReserve == null || ptr.compareTo(oldestWALPointerToReserve) < 0)
            oldestWALPointerToReserve = ptr;
    }
    if (cctx.wal().reserve(oldestWALPointerToReserve)) {
        reservedForPreloading.set(oldestWALPointerToReserve);
        return true;
    } else
        return false;
}

Example 14 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class GridCacheOffheapManager method addPartition.

/**
 * @param part Local partition.
 * @param map Map to add values to.
 * @param metaPageAddr Meta page address
 * @param io Page Meta IO
 * @param grpId Cache Group ID.
 * @param currAllocatedPageCnt total number of pages allocated for partition <code>[partition, grpId]</code>
 */
private static boolean addPartition(GridDhtLocalPartition part, PartitionAllocationMap map, long metaPageAddr, PageMetaIO io, int grpId, int partId, int currAllocatedPageCnt, long partSize) {
    if (part != null) {
        boolean reserved = part.reserve();
        if (!reserved)
            return false;
    } else
        assert partId == PageIdAllocator.INDEX_PARTITION : partId;
    assert PageIO.getPageId(metaPageAddr) != 0;
    int lastAllocatedPageCnt = io.getLastAllocatedPageCount(metaPageAddr);
    int curPageCnt = partSize == 0 ? 0 : currAllocatedPageCnt;
    map.put(new GroupPartitionId(grpId, partId), new PagesAllocationRange(lastAllocatedPageCnt, curPageCnt));
    return true;
}

Example 15 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class IgniteClusterSnapshotSelfTest method blockingLocalSnapshotSender.

/**
 * @param ignite Ignite instance.
 * @param started Latch will be released when delta partition processing starts.
 * @param blocked Latch to await delta partition processing.
 * @return Factory which produces local snapshot senders.
 */
private Function<String, SnapshotSender> blockingLocalSnapshotSender(IgniteEx ignite, CountDownLatch started, CountDownLatch blocked) {
    Function<String, SnapshotSender> old = snp(ignite).localSnapshotSenderFactory();
    return (snpName) -> new DelegateSnapshotSender(log, snp(ignite).snapshotExecutorService(), old.apply(snpName)) {

        @Override
        public void sendDelta0(File delta, String cacheDirName, GroupPartitionId pair) {
            if (log.isInfoEnabled())
                log.info("Processing delta file has been blocked: " + delta.getName());
            started.countDown();
            try {
                U.await(blocked, TIMEOUT, TimeUnit.MILLISECONDS);
                if (log.isInfoEnabled())
                    log.info("Latch released. Processing delta file continued: " + delta.getName());
                super.sendDelta0(delta, cacheDirName, pair);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteException("Interrupted by node stop", e);
            }
        }
    };
}