Examples with ClusterTopologyCheckedException org.apache.ignite.internal.cluster.ClusterTopologyCheckedException used on opensource projects

Example 26 with ClusterTopologyCheckedException

use of org.apache.ignite.internal.cluster.ClusterTopologyCheckedException in project ignite by apache.

the class GridCacheTxRecoveryFuture method proceedPrepare.

/**
 * Process prepare after local check.
 */
private void proceedPrepare() {
    for (Map.Entry<UUID, Collection<UUID>> entry : txNodes.entrySet()) {
        UUID nodeId = entry.getKey();
        // Skipping iteration when local node is one of tx's primary.
        if (!nodes.containsKey(nodeId) && nodeId.equals(cctx.localNodeId()))
            continue;
        if (failedNodeIds.contains(nodeId)) {
            for (UUID id : entry.getValue()) {
                // Skip backup node if it is local node or if it is also was mapped as primary.
                if (txNodes.containsKey(id) || id.equals(cctx.localNodeId()))
                    continue;
                MiniFuture fut = new MiniFuture(id);
                add(fut);
                GridCacheTxRecoveryRequest req = new GridCacheTxRecoveryRequest(tx, nodeTransactions(id), false, futureId(), fut.futureId(), tx.activeCachesDeploymentEnabled());
                try {
                    cctx.io().send(id, req, tx.ioPolicy());
                    if (msgLog.isInfoEnabled()) {
                        msgLog.info("Recovery fut, sent request to backup [txId=" + tx.nearXidVersion() + ", dhtTxId=" + tx.xidVersion() + ", node=" + id + ']');
                    }
                } catch (ClusterTopologyCheckedException ignored) {
                    fut.onNodeLeft(id);
                } catch (IgniteCheckedException e) {
                    if (msgLog.isInfoEnabled()) {
                        msgLog.info("Recovery fut, failed to send request to backup [txId=" + tx.nearXidVersion() + ", dhtTxId=" + tx.xidVersion() + ", node=" + id + ", err=" + e + ']');
                    }
                    fut.onError(e);
                    break;
                }
            }
        } else {
            MiniFuture fut = new MiniFuture(nodeId);
            add(fut);
            GridCacheTxRecoveryRequest req = new GridCacheTxRecoveryRequest(tx, nodeTransactions(nodeId), false, futureId(), fut.futureId(), tx.activeCachesDeploymentEnabled());
            try {
                cctx.io().send(nodeId, req, tx.ioPolicy());
                if (msgLog.isInfoEnabled()) {
                    msgLog.info("Recovery fut, sent request to primary [txId=" + tx.nearXidVersion() + ", dhtTxId=" + tx.xidVersion() + ", node=" + nodeId + ']');
                }
            } catch (ClusterTopologyCheckedException ignored) {
                fut.onNodeLeft(nodeId);
            } catch (IgniteCheckedException e) {
                if (msgLog.isInfoEnabled()) {
                    msgLog.info("Recovery fut, failed to send request to primary [txId=" + tx.nearXidVersion() + ", dhtTxId=" + tx.xidVersion() + ", node=" + nodeId + ", err=" + e + ']');
                }
                fut.onError(e);
                break;
            }
        }
    }
    markInitialized();
}

Example 27 with ClusterTopologyCheckedException

use of org.apache.ignite.internal.cluster.ClusterTopologyCheckedException in project ignite by apache.

the class GridDhtPartitionsExchangeFuture method onBecomeCoordinator.

/**
 * @param newCrdFut Coordinator initialization future.
 */
private void onBecomeCoordinator(InitNewCoordinatorFuture newCrdFut) {
    boolean allRcvd = false;
    cctx.exchange().onCoordinatorInitialized();
    if (newCrdFut.restoreState()) {
        GridDhtPartitionsFullMessage fullMsg = newCrdFut.fullMessage();
        assert msgs.isEmpty() : msgs;
        if (fullMsg != null) {
            if (log.isInfoEnabled()) {
                log.info("New coordinator restored state [ver=" + initialVersion() + ", resVer=" + fullMsg.resultTopologyVersion() + ']');
            }
            synchronized (mux) {
                state = ExchangeLocalState.DONE;
                finishState = new FinishState(crd.id(), fullMsg.resultTopologyVersion(), fullMsg);
            }
            fullMsg.exchangeId(exchId);
            processFullMessage(false, null, fullMsg);
            Map<ClusterNode, GridDhtPartitionsSingleMessage> msgs = newCrdFut.messages();
            if (!F.isEmpty(msgs)) {
                Map<Integer, CacheGroupAffinityMessage> joinedNodeAff = new ConcurrentHashMap<>();
                // Reserve at least 2 threads for system operations.
                int parallelismLvl = U.availableThreadCount(cctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2);
                try {
                    U.doInParallel(parallelismLvl, cctx.kernalContext().pools().getSystemExecutorService(), msgs.entrySet(), entry -> {
                        this.msgs.put(entry.getKey().id(), entry.getValue());
                        GridDhtPartitionsSingleMessage msg = entry.getValue();
                        Collection<Integer> affReq = msg.cacheGroupsAffinityRequest();
                        if (!F.isEmpty(affReq)) {
                            CacheGroupAffinityMessage.createAffinityMessages(cctx, fullMsg.resultTopologyVersion(), affReq, joinedNodeAff);
                        }
                        return null;
                    });
                } catch (IgniteCheckedException e) {
                    throw new IgniteException(e);
                }
                Map<UUID, GridDhtPartitionsSingleMessage> mergedJoins = newCrdFut.mergedJoinExchangeMessages();
                if (log.isInfoEnabled()) {
                    log.info("New coordinator sends full message [ver=" + initialVersion() + ", resVer=" + fullMsg.resultTopologyVersion() + ", nodes=" + F.nodeIds(msgs.keySet()) + ", mergedJoins=" + (mergedJoins != null ? mergedJoins.keySet() : null) + ']');
                }
                sendAllPartitions(fullMsg, msgs.keySet(), mergedJoins, joinedNodeAff);
            }
            return;
        } else {
            if (log.isInfoEnabled())
                log.info("New coordinator restore state finished [ver=" + initialVersion() + ']');
            for (Map.Entry<ClusterNode, GridDhtPartitionsSingleMessage> e : newCrdFut.messages().entrySet()) {
                GridDhtPartitionsSingleMessage msg = e.getValue();
                if (!msg.client()) {
                    msgs.put(e.getKey().id(), e.getValue());
                    if (dynamicCacheStartExchange() && msg.getError() != null)
                        exchangeGlobalExceptions.put(e.getKey().id(), msg.getError());
                    updatePartitionSingleMap(e.getKey().id(), msg);
                }
            }
        }
        allRcvd = true;
        synchronized (mux) {
            // Do not process messages.
            remaining.clear();
            assert crd != null && crd.isLocal();
            state = ExchangeLocalState.CRD;
            assert mergedJoinExchMsgs == null;
        }
    } else {
        Set<UUID> remaining0 = null;
        synchronized (mux) {
            assert crd != null && crd.isLocal();
            state = ExchangeLocalState.CRD;
            assert mergedJoinExchMsgs == null;
            if (log.isInfoEnabled()) {
                log.info("New coordinator initialization finished [ver=" + initialVersion() + ", remaining=" + remaining + ']');
            }
            if (!remaining.isEmpty())
                remaining0 = new HashSet<>(remaining);
        }
        if (remaining0 != null) {
            // It is possible that some nodes finished exchange with previous coordinator.
            GridDhtPartitionsSingleRequest req = new GridDhtPartitionsSingleRequest(exchId);
            for (UUID nodeId : remaining0) {
                try {
                    if (!pendingSingleMsgs.containsKey(nodeId)) {
                        if (log.isInfoEnabled()) {
                            log.info("New coordinator sends request [ver=" + initialVersion() + ", node=" + nodeId + ']');
                        }
                        cctx.io().send(nodeId, req, SYSTEM_POOL);
                    }
                } catch (ClusterTopologyCheckedException ignored) {
                    if (log.isDebugEnabled())
                        log.debug("Node left during partition exchange [nodeId=" + nodeId + ", exchId=" + exchId + ']');
                } catch (IgniteCheckedException e) {
                    U.error(log, "Failed to request partitions from node: " + nodeId, e);
                }
            }
            for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : pendingSingleMsgs.entrySet()) {
                if (log.isInfoEnabled()) {
                    log.info("New coordinator process pending message [ver=" + initialVersion() + ", node=" + m.getKey() + ']');
                }
                processSingleMessage(m.getKey(), m.getValue());
            }
        }
    }
    if (allRcvd) {
        awaitSingleMapUpdates();
        onAllReceived(newCrdFut.messages().keySet());
    }
}

Example 28 with ClusterTopologyCheckedException

use of org.apache.ignite.internal.cluster.ClusterTopologyCheckedException in project ignite by apache.

the class GridDhtPartitionsExchangeFuture method sendAllPartitions.

/**
 * @param fullMsg Message to send.
 * @param nodes Target Nodes.
 * @param mergedJoinExchMsgs Messages received from merged 'join node' exchanges.
 * @param affinityForJoinedNodes Affinity if was requested by some nodes.
 */
private void sendAllPartitions(GridDhtPartitionsFullMessage fullMsg, Collection<ClusterNode> nodes, Map<UUID, GridDhtPartitionsSingleMessage> mergedJoinExchMsgs, Map<Integer, CacheGroupAffinityMessage> affinityForJoinedNodes) {
    assert !nodes.contains(cctx.localNode());
    if (log.isTraceEnabled()) {
        log.trace("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) + ", exchId=" + exchId + ", msg=" + fullMsg + ']');
    }
    // Find any single message with affinity request. This request exists only for newly joined nodes.
    Optional<GridDhtPartitionsSingleMessage> singleMsgWithAffinityReq = nodes.stream().flatMap(node -> Optional.ofNullable(msgs.get(node.id())).filter(singleMsg -> singleMsg.cacheGroupsAffinityRequest() != null).map(Stream::of).orElse(Stream.empty())).findAny();
    // Prepare full message for newly joined nodes with affinity request.
    final GridDhtPartitionsFullMessage fullMsgWithAffinity = singleMsgWithAffinityReq.filter(singleMessage -> affinityForJoinedNodes != null).map(singleMessage -> fullMsg.copy().joinedNodeAffinity(affinityForJoinedNodes)).orElse(null);
    // Prepare and send full messages for given nodes.
    nodes.stream().map(node -> {
        // No joined nodes, just send a regular full message.
        if (fullMsgWithAffinity == null)
            return new T2<>(node, fullMsg);
        return new T2<>(node, // If single message contains affinity request, use special full message for such single messages.
        Optional.ofNullable(msgs.get(node.id())).filter(singleMsg -> singleMsg.cacheGroupsAffinityRequest() != null).map(singleMsg -> fullMsgWithAffinity).orElse(fullMsg));
    }).map(nodeAndMsg -> {
        ClusterNode node = nodeAndMsg.get1();
        GridDhtPartitionsFullMessage fullMsgToSend = nodeAndMsg.get2();
        // If exchange has merged, use merged version of exchange id.
        GridDhtPartitionExchangeId sndExchId = mergedJoinExchMsgs != null ? Optional.ofNullable(mergedJoinExchMsgs.get(node.id())).map(GridDhtPartitionsAbstractMessage::exchangeId).orElse(exchangeId()) : exchangeId();
        if (sndExchId != null && !sndExchId.equals(exchangeId())) {
            GridDhtPartitionsFullMessage fullMsgWithUpdatedExchangeId = fullMsgToSend.copy();
            fullMsgWithUpdatedExchangeId.exchangeId(sndExchId);
            return new T2<>(node, fullMsgWithUpdatedExchangeId);
        }
        return new T2<>(node, fullMsgToSend);
    }).forEach(nodeAndMsg -> {
        ClusterNode node = nodeAndMsg.get1();
        GridDhtPartitionsFullMessage fullMsgToSend = nodeAndMsg.get2();
        try {
            cctx.io().send(node, fullMsgToSend, SYSTEM_POOL);
        } catch (ClusterTopologyCheckedException e) {
            if (log.isDebugEnabled())
                log.debug("Failed to send partitions, node failed: " + node);
        } catch (IgniteCheckedException e) {
            U.error(log, "Failed to send partitions [node=" + node + ']', e);
        }
    });
}

Example 29 with ClusterTopologyCheckedException

use of org.apache.ignite.internal.cluster.ClusterTopologyCheckedException in project ignite by apache.

the class GridDhtPartitionDemander method handleSupplyMessage.

/**
 * Handles supply message from {@code nodeId} with specified {@code topicId}.
 *
 * Supply message contains entries to populate rebalancing partitions.
 *
 * There is a cyclic process:
 * Populate rebalancing partitions with entries from Supply message.
 * If not all partitions specified in {@link #rebalanceFut} were rebalanced or marked as missed
 * send new Demand message to request next batch of entries.
 *
 * @param nodeId Node id.
 * @param supplyMsg Supply message.
 */
public void handleSupplyMessage(final UUID nodeId, final GridDhtPartitionSupplyMessage supplyMsg) {
    AffinityTopologyVersion topVer = supplyMsg.topologyVersion();
    RebalanceFuture fut = rebalanceFut;
    ClusterNode node = ctx.node(nodeId);
    fut.cancelLock.readLock().lock();
    try {
        String errMsg = null;
        if (fut.isDone())
            errMsg = "rebalance completed";
        else if (node == null)
            errMsg = "supplier has left cluster";
        else if (!rebalanceFut.isActual(supplyMsg.rebalanceId()))
            errMsg = "topology changed";
        if (errMsg != null) {
            if (log.isDebugEnabled()) {
                log.debug("Supply message has been ignored (" + errMsg + ") [" + demandRoutineInfo(nodeId, supplyMsg) + ']');
            }
            return;
        }
        if (log.isDebugEnabled())
            log.debug("Received supply message [" + demandRoutineInfo(nodeId, supplyMsg) + ']');
        // Check whether there were error during supplying process.
        Throwable msgExc = null;
        final GridDhtPartitionTopology top = grp.topology();
        if (supplyMsg.classError() != null)
            msgExc = supplyMsg.classError();
        else if (supplyMsg.error() != null)
            msgExc = supplyMsg.error();
        if (msgExc != null) {
            GridDhtPartitionMap partMap = top.localPartitionMap();
            Set<Integer> unstableParts = supplyMsg.infos().keySet().stream().filter(p -> partMap.get(p) == MOVING).collect(Collectors.toSet());
            U.error(log, "Rebalancing routine has failed, some partitions could be unavailable for reading" + " [" + demandRoutineInfo(nodeId, supplyMsg) + ", unavailablePartitions=" + S.compact(unstableParts) + ']', msgExc);
            fut.error(nodeId);
            return;
        }
        fut.receivedBytes.addAndGet(supplyMsg.messageSize());
        if (grp.sharedGroup()) {
            for (GridCacheContext cctx : grp.caches()) {
                if (cctx.statisticsEnabled()) {
                    long keysCnt = supplyMsg.keysForCache(cctx.cacheId());
                    if (keysCnt != -1)
                        cctx.cache().metrics0().onRebalancingKeysCountEstimateReceived(keysCnt);
                    // Can not be calculated per cache.
                    cctx.cache().metrics0().onRebalanceBatchReceived(supplyMsg.messageSize());
                }
            }
        } else {
            GridCacheContext cctx = grp.singleCacheContext();
            if (cctx.statisticsEnabled()) {
                if (supplyMsg.estimatedKeysCount() != -1)
                    cctx.cache().metrics0().onRebalancingKeysCountEstimateReceived(supplyMsg.estimatedKeysCount());
                cctx.cache().metrics0().onRebalanceBatchReceived(supplyMsg.messageSize());
            }
        }
        try {
            AffinityAssignment aff = grp.affinity().cachedAffinity(topVer);
            // Preload.
            for (Map.Entry<Integer, CacheEntryInfoCollection> e : supplyMsg.infos().entrySet()) {
                int p = e.getKey();
                if (aff.get(p).contains(ctx.localNode())) {
                    GridDhtLocalPartition part;
                    try {
                        part = top.localPartition(p, topVer, true);
                    } catch (GridDhtInvalidPartitionException err) {
                        assert !topVer.equals(top.lastTopologyChangeVersion());
                        if (log.isDebugEnabled()) {
                            log.debug("Failed to get partition for rebalancing [" + "grp=" + grp.cacheOrGroupName() + ", err=" + err + ", p=" + p + ", topVer=" + topVer + ", lastTopVer=" + top.lastTopologyChangeVersion() + ']');
                        }
                        continue;
                    }
                    assert part != null;
                    boolean last = supplyMsg.last().containsKey(p);
                    if (part.state() == MOVING) {
                        boolean reserved = part.reserve();
                        assert reserved : "Failed to reserve partition [igniteInstanceName=" + ctx.igniteInstanceName() + ", grp=" + grp.cacheOrGroupName() + ", part=" + part + ']';
                        part.beforeApplyBatch(last);
                        try {
                            long[] byteRcv = { 0 };
                            GridIterableAdapter<GridCacheEntryInfo> infosWrap = new GridIterableAdapter<>(new IteratorWrapper<GridCacheEntryInfo>(e.getValue().infos().iterator()) {

                                /**
                                 * {@inheritDoc}
                                 */
                                @Override
                                public GridCacheEntryInfo nextX() throws IgniteCheckedException {
                                    GridCacheEntryInfo i = super.nextX();
                                    byteRcv[0] += i.marshalledSize(ctx.cacheObjectContext(i.cacheId()));
                                    return i;
                                }
                            });
                            try {
                                if (grp.mvccEnabled())
                                    mvccPreloadEntries(topVer, node, p, infosWrap);
                                else {
                                    preloadEntries(topVer, part, infosWrap);
                                    rebalanceFut.onReceivedKeys(p, e.getValue().infos().size(), node);
                                }
                            } catch (GridDhtInvalidPartitionException ignored) {
                                if (log.isDebugEnabled())
                                    log.debug("Partition became invalid during rebalancing (will ignore): " + p);
                            }
                            fut.processed.get(p).increment();
                            fut.onReceivedBytes(p, byteRcv[0], node);
                            // If message was last for this partition, then we take ownership.
                            if (last)
                                ownPartition(fut, p, nodeId, supplyMsg);
                        } finally {
                            part.release();
                        }
                    } else {
                        if (last)
                            fut.partitionDone(nodeId, p, false);
                        if (log.isDebugEnabled())
                            log.debug("Skipping rebalancing partition (state is not MOVING): " + '[' + demandRoutineInfo(nodeId, supplyMsg) + ", p=" + p + ']');
                    }
                } else {
                    fut.partitionDone(nodeId, p, false);
                    if (log.isDebugEnabled())
                        log.debug("Skipping rebalancing partition (affinity changed): " + '[' + demandRoutineInfo(nodeId, supplyMsg) + ", p=" + p + ']');
                }
            }
            // Only request partitions based on latest topology version.
            for (Integer miss : supplyMsg.missed()) {
                if (aff.get(miss).contains(ctx.localNode()))
                    fut.partitionMissed(nodeId, miss);
            }
            for (Integer miss : supplyMsg.missed()) fut.partitionDone(nodeId, miss, false);
            GridDhtPartitionDemandMessage d = new GridDhtPartitionDemandMessage(supplyMsg.rebalanceId(), supplyMsg.topologyVersion(), grp.groupId());
            d.timeout(grp.preloader().timeout());
            if (!fut.isDone()) {
                // Send demand message.
                try {
                    ctx.io().sendOrderedMessage(node, d.topic(), d.convertIfNeeded(node.version()), grp.ioPolicy(), grp.preloader().timeout());
                    if (log.isDebugEnabled())
                        log.debug("Send next demand message [" + demandRoutineInfo(nodeId, supplyMsg) + "]");
                } catch (ClusterTopologyCheckedException e) {
                    if (log.isDebugEnabled())
                        log.debug("Supplier has left [" + demandRoutineInfo(nodeId, supplyMsg) + ", errMsg=" + e.getMessage() + ']');
                }
            } else {
                if (log.isDebugEnabled())
                    log.debug("Will not request next demand message [" + demandRoutineInfo(nodeId, supplyMsg) + ", rebalanceFuture=" + fut + ']');
            }
        } catch (IgniteSpiException | IgniteCheckedException e) {
            fut.error(nodeId);
            LT.error(log, e, "Error during rebalancing [" + demandRoutineInfo(nodeId, supplyMsg) + ", err=" + e + ']');
        }
    } finally {
        fut.cancelLock.readLock().unlock();
    }
}

Example 30 with ClusterTopologyCheckedException

use of org.apache.ignite.internal.cluster.ClusterTopologyCheckedException in project ignite by apache.

the class GridDhtForceKeysFuture method map.

/**
 * @param keys Keys.
 * @param exc Exclude nodes.
 * @return {@code True} if some mapping was added.
 */
private boolean map(Iterable<KeyCacheObject> keys, Collection<ClusterNode> exc) {
    Map<ClusterNode, Set<KeyCacheObject>> mappings = null;
    for (KeyCacheObject key : keys) mappings = map(key, mappings, exc);
    if (isDone())
        return false;
    boolean ret = false;
    if (mappings != null) {
        // Should not happen when MVCC enabled.
        assert !cctx.mvccEnabled();
        ClusterNode loc = cctx.localNode();
        int curTopVer = topCntr.get();
        if (!cctx.dht().addFuture(this)) {
            assert isDone() : this;
            return false;
        }
        trackable = true;
        // Create mini futures.
        for (Map.Entry<ClusterNode, Set<KeyCacheObject>> mapped : mappings.entrySet()) {
            ClusterNode n = mapped.getKey();
            Set<KeyCacheObject> mappedKeys = mapped.getValue();
            int cnt = F.size(mappedKeys);
            if (cnt > 0) {
                ret = true;
                MiniFuture fut = new MiniFuture(n, mappedKeys, curTopVer, exc);
                GridDhtForceKeysRequest req = new GridDhtForceKeysRequest(cctx.cacheId(), futId, fut.miniId(), mappedKeys, topVer, cctx.deploymentEnabled());
                try {
                    // Append new future.
                    add(fut);
                    assert !n.id().equals(loc.id());
                    if (log.isTraceEnabled())
                        log.trace("Sending force key request [cacheName=" + cctx.name() + "node=" + n.id() + ", req=" + req + ']');
                    cctx.io().send(n, req, cctx.ioPolicy());
                } catch (IgniteCheckedException e) {
                    // Fail the whole thing.
                    if (e instanceof ClusterTopologyCheckedException)
                        fut.onResult();
                    else if (!cctx.kernalContext().isStopping())
                        fut.onResult(e);
                }
            }
        }
    }
    return ret;
}