Examples with ChangeGlobalStateMessage org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage used on opensource projects

Example 1 with ChangeGlobalStateMessage

use of org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage in project ignite by apache.

the class GridCachePartitionExchangeManager method onDiscoveryEvent.

/**
 * @param evt Event.
 * @param cache Discovery data cache.
 */
private void onDiscoveryEvent(DiscoveryEvent evt, DiscoCache cache) {
    ClusterNode loc = cctx.localNode();
    assert evt.type() == EVT_NODE_JOINED || evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED || evt.type() == EVT_DISCOVERY_CUSTOM_EVT;
    final ClusterNode n = evt.eventNode();
    GridDhtPartitionExchangeId exchId = null;
    GridDhtPartitionsExchangeFuture exchFut = null;
    if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
        assert evt.type() != EVT_NODE_JOINED || n.isLocal() || n.order() > loc.order() : "Node joined with smaller-than-local " + "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']';
        exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
        ExchangeActions exchActs = null;
        if (evt.type() == EVT_NODE_JOINED && evt.eventNode().isLocal()) {
            LocalJoinCachesContext locJoinCtx = cctx.cache().localJoinCachesContext();
            if (locJoinCtx != null) {
                exchActs = new ExchangeActions();
                exchActs.localJoinContext(locJoinCtx);
            }
        }
        exchFut = exchangeFuture(exchId, evt, cache, exchActs, null);
    } else {
        DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent) evt).customMessage();
        if (customMsg instanceof ChangeGlobalStateMessage) {
            ChangeGlobalStateMessage stateChangeMsg = (ChangeGlobalStateMessage) customMsg;
            ExchangeActions exchActions = stateChangeMsg.exchangeActions();
            if (exchActions != null) {
                exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
                exchFut = exchangeFuture(exchId, evt, cache, exchActions, null);
            }
        } else if (customMsg instanceof DynamicCacheChangeBatch) {
            DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch) customMsg;
            ExchangeActions exchActions = batch.exchangeActions();
            if (exchActions != null) {
                exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
                exchFut = exchangeFuture(exchId, evt, cache, exchActions, null);
            }
        } else if (customMsg instanceof CacheAffinityChangeMessage) {
            CacheAffinityChangeMessage msg = (CacheAffinityChangeMessage) customMsg;
            if (msg.exchangeId() == null) {
                if (msg.exchangeNeeded()) {
                    exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
                    exchFut = exchangeFuture(exchId, evt, cache, null, msg);
                }
            } else if (msg.exchangeId().topologyVersion().topologyVersion() >= cctx.discovery().localJoinEvent().topologyVersion())
                exchangeFuture(msg.exchangeId(), null, null, null, null).onAffinityChangeMessage(evt.eventNode(), msg);
        } else if (customMsg instanceof SnapshotDiscoveryMessage && ((SnapshotDiscoveryMessage) customMsg).needExchange()) {
            exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
            exchFut = exchangeFuture(exchId, evt, null, null, null);
        } else if (customMsg instanceof WalStateAbstractMessage && ((WalStateAbstractMessage) customMsg).needExchange()) {
            exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
            exchFut = exchangeFuture(exchId, evt, null, null, null);
        } else {
            // Process event as custom discovery task if needed.
            CachePartitionExchangeWorkerTask task = cctx.cache().exchangeTaskForCustomDiscoveryMessage(customMsg);
            if (task != null)
                exchWorker.addCustomTask(task);
        }
    }
    if (exchId != null) {
        if (log.isDebugEnabled())
            log.debug("Discovery event (will start exchange): " + exchId);
        // Event callback - without this callback future will never complete.
        exchFut.onEvent(exchId, evt, cache);
        // Start exchange process.
        addFuture(exchFut);
    } else {
        if (log.isDebugEnabled())
            log.debug("Do not start exchange for discovery event: " + evt);
    }
    notifyNodeFail(evt);
    // Notify indexing engine about node leave so that we can re-map coordinator accordingly.
    if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) {
        exchWorker.addCustomTask(new SchemaNodeLeaveExchangeWorkerTask(evt.eventNode()));
        exchWorker.addCustomTask(new WalStateNodeLeaveExchangeTask(evt.eventNode()));
    }
}

Example 2 with ChangeGlobalStateMessage

use of org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage in project ignite by apache.

the class GridDiscoveryManager method start.

/**
 * {@inheritDoc}
 */
@Override
public void start() throws IgniteCheckedException {
    long totSysMemory = -1;
    try {
        totSysMemory = U.<Long>property(os, "totalPhysicalMemorySize");
    } catch (RuntimeException ignored) {
    // No-op.
    }
    ctx.addNodeAttribute(ATTR_PHY_RAM, totSysMemory);
    ctx.addNodeAttribute(ATTR_OFFHEAP_SIZE, requiredOffheap());
    ctx.addNodeAttribute(ATTR_DATA_REGIONS_OFFHEAP_SIZE, configuredOffheap());
    DiscoverySpi spi = getSpi();
    discoOrdered = discoOrdered();
    histSupported = historySupported();
    isLocDaemon = ctx.isDaemon();
    hasRslvrs = !ctx.config().isClientMode() && !F.isEmpty(ctx.config().getSegmentationResolvers());
    segChkFreq = ctx.config().getSegmentCheckFrequency();
    if (hasRslvrs) {
        if (segChkFreq < 0)
            throw new IgniteCheckedException("Segment check frequency cannot be negative: " + segChkFreq);
        if (segChkFreq > 0 && segChkFreq < 2000)
            U.warn(log, "Configuration parameter 'segmentCheckFrequency' is too low " + "(at least 2000 ms recommended): " + segChkFreq);
        int segResAttemp = ctx.config().getSegmentationResolveAttempts();
        if (segResAttemp < 1)
            throw new IgniteCheckedException("Segment resolve attempts cannot be negative or zero: " + segResAttemp);
        checkSegmentOnStart();
    }
    metricsUpdateTask = ctx.timeout().schedule(new MetricsUpdater(), METRICS_UPDATE_FREQ, METRICS_UPDATE_FREQ);
    spi.setMetricsProvider(createMetricsProvider());
    if (ctx.security().enabled()) {
        if (isSecurityCompatibilityMode())
            ctx.addNodeAttribute(ATTR_SECURITY_COMPATIBILITY_MODE, true);
        spi.setAuthenticator(new DiscoverySpiNodeAuthenticator() {

            @Override
            public SecurityContext authenticateNode(ClusterNode node, SecurityCredentials cred) {
                try {
                    return ctx.security().authenticateNode(node, cred);
                } catch (IgniteCheckedException e) {
                    throw U.convertException(e);
                }
            }

            @Override
            public boolean isGlobalNodeAuthentication() {
                return ctx.security().isGlobalNodeAuthentication();
            }
        });
    }
    spi.setListener(new DiscoverySpiListener() {

        private long gridStartTime;

        /**
         * {@inheritDoc}
         */
        @Override
        public void onLocalNodeInitialized(ClusterNode locNode) {
            for (IgniteInClosure<ClusterNode> lsnr : locNodeInitLsnrs) lsnr.apply(locNode);
            if (locNode instanceof TcpDiscoveryNode) {
                final TcpDiscoveryNode node = (TcpDiscoveryNode) locNode;
                if (consistentId != null)
                    node.setConsistentId(consistentId);
            }
        }

        @Override
        public void onDiscovery(final int type, final long topVer, final ClusterNode node, final Collection<ClusterNode> topSnapshot, final Map<Long, Collection<ClusterNode>> snapshots, @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
            synchronized (discoEvtMux) {
                onDiscovery0(type, topVer, node, topSnapshot, snapshots, spiCustomMsg);
            }
        }

        /**
         * @param type Event type.
         * @param topVer Event topology version.
         * @param node Event node.
         * @param topSnapshot Topology snapsjot.
         * @param snapshots Topology snapshots history.
         * @param spiCustomMsg Custom event.
         */
        private void onDiscovery0(final int type, final long topVer, final ClusterNode node, final Collection<ClusterNode> topSnapshot, final Map<Long, Collection<ClusterNode>> snapshots, @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
            DiscoveryCustomMessage customMsg = spiCustomMsg == null ? null : ((CustomMessageWrapper) spiCustomMsg).delegate();
            if (skipMessage(type, customMsg))
                return;
            final ClusterNode locNode = localNode();
            if (snapshots != null)
                topHist = snapshots;
            boolean verChanged;
            if (type == EVT_NODE_METRICS_UPDATED)
                verChanged = false;
            else {
                if (type != EVT_NODE_SEGMENTED && type != EVT_CLIENT_NODE_DISCONNECTED && type != EVT_CLIENT_NODE_RECONNECTED && type != EVT_DISCOVERY_CUSTOM_EVT) {
                    minorTopVer = 0;
                    verChanged = true;
                } else
                    verChanged = false;
            }
            if (type == EVT_NODE_FAILED || type == EVT_NODE_LEFT) {
                for (DiscoCache c : discoCacheHist.values()) c.updateAlives(node);
                updateClientNodes(node.id());
            }
            boolean locJoinEvt = type == EVT_NODE_JOINED && node.id().equals(locNode.id());
            ChangeGlobalStateFinishMessage stateFinishMsg = null;
            if (type == EVT_NODE_FAILED || type == EVT_NODE_LEFT)
                stateFinishMsg = ctx.state().onNodeLeft(node);
            final AffinityTopologyVersion nextTopVer;
            if (type == EVT_DISCOVERY_CUSTOM_EVT) {
                assert customMsg != null;
                boolean incMinorTopVer;
                if (customMsg instanceof ChangeGlobalStateMessage) {
                    incMinorTopVer = ctx.state().onStateChangeMessage(new AffinityTopologyVersion(topVer, minorTopVer), (ChangeGlobalStateMessage) customMsg, discoCache());
                } else if (customMsg instanceof ChangeGlobalStateFinishMessage) {
                    ctx.state().onStateFinishMessage((ChangeGlobalStateFinishMessage) customMsg);
                    Snapshot snapshot = topSnap.get();
                    // Topology version does not change, but need create DiscoCache with new state.
                    DiscoCache discoCache = snapshot.discoCache.copy(snapshot.topVer, ctx.state().clusterState());
                    topSnap.set(new Snapshot(snapshot.topVer, discoCache));
                    incMinorTopVer = false;
                } else {
                    incMinorTopVer = ctx.cache().onCustomEvent(customMsg, new AffinityTopologyVersion(topVer, minorTopVer), node);
                }
                if (incMinorTopVer) {
                    minorTopVer++;
                    verChanged = true;
                }
                nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer);
            } else
                nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer);
            ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState());
            if (type == EVT_DISCOVERY_CUSTOM_EVT) {
                for (Class cls = customMsg.getClass(); cls != null; cls = cls.getSuperclass()) {
                    List<CustomEventListener<DiscoveryCustomMessage>> list = customEvtLsnrs.get(cls);
                    if (list != null) {
                        for (CustomEventListener<DiscoveryCustomMessage> lsnr : list) {
                            try {
                                lsnr.onCustomEvent(nextTopVer, node, customMsg);
                            } catch (Exception e) {
                                U.error(log, "Failed to notify direct custom event listener: " + customMsg, e);
                            }
                        }
                    }
                }
            }
            DiscoCache discoCache;
            // event notifications, since SPI notifies manager about all events from this listener.
            if (verChanged) {
                Snapshot snapshot = topSnap.get();
                if (customMsg == null) {
                    discoCache = createDiscoCache(nextTopVer, ctx.state().clusterState(), locNode, topSnapshot);
                } else if (customMsg instanceof ChangeGlobalStateMessage) {
                    discoCache = createDiscoCache(nextTopVer, ctx.state().pendingState((ChangeGlobalStateMessage) customMsg), locNode, topSnapshot);
                } else
                    discoCache = customMsg.createDiscoCache(GridDiscoveryManager.this, nextTopVer, snapshot.discoCache);
                discoCacheHist.put(nextTopVer, discoCache);
                assert snapshot.topVer.compareTo(nextTopVer) < 0 : "Topology version out of order [this.topVer=" + topSnap + ", topVer=" + topVer + ", node=" + node + ", nextTopVer=" + nextTopVer + ", evt=" + U.gridEventName(type) + ']';
                topSnap.set(new Snapshot(nextTopVer, discoCache));
            } else
                // Current version.
                discoCache = discoCache();
            final DiscoCache discoCache0 = discoCache;
            // If this is a local join event, just save it and do not notify listeners.
            if (locJoinEvt) {
                if (gridStartTime == 0)
                    gridStartTime = getSpi().getGridStartTime();
                topSnap.set(new Snapshot(nextTopVer, discoCache));
                startLatch.countDown();
                DiscoveryEvent discoEvt = new DiscoveryEvent();
                discoEvt.node(ctx.discovery().localNode());
                discoEvt.eventNode(node);
                discoEvt.type(EVT_NODE_JOINED);
                discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(topSnapshot, FILTER_NOT_DAEMON)));
                discoWrk.discoCache = discoCache;
                if (!isLocDaemon && !ctx.clientDisconnected()) {
                    ctx.cache().context().exchange().onLocalJoin(discoEvt, discoCache);
                    ctx.authentication().onLocalJoin();
                }
                IgniteInternalFuture<Boolean> transitionWaitFut = ctx.state().onLocalJoin(discoCache);
                locJoin.onDone(new DiscoveryLocalJoinData(discoEvt, discoCache, transitionWaitFut, ctx.state().clusterState().active()));
                return;
            } else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                assert locNode.isClient() : locNode;
                assert node.isClient() : node;
                ((IgniteKernal) ctx.grid()).onDisconnected();
                locJoin = new GridFutureAdapter<>();
                registeredCaches.clear();
                registeredCacheGrps.clear();
                for (AffinityTopologyVersion histVer : discoCacheHist.keySet()) {
                    Object rmvd = discoCacheHist.remove(histVer);
                    assert rmvd != null : histVer;
                }
                topHist.clear();
                topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO, createDiscoCache(AffinityTopologyVersion.ZERO, ctx.state().clusterState(), locNode, Collections.singleton(locNode))));
            } else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                assert locNode.isClient() : locNode;
                assert node.isClient() : node;
                boolean clusterRestarted = gridStartTime != getSpi().getGridStartTime();
                gridStartTime = getSpi().getGridStartTime();
                ((IgniteKernal) ctx.grid()).onReconnected(clusterRestarted);
                ctx.cache().context().exchange().onLocalJoin(localJoinEvent(), discoCache);
                ctx.cluster().clientReconnectFuture().listen(new CI1<IgniteFuture<?>>() {

                    @Override
                    public void apply(IgniteFuture<?> fut) {
                        try {
                            fut.get();
                            discoWrk.addEvent(type, nextTopVer, node, discoCache0, topSnapshot, null);
                        } catch (IgniteException ignore) {
                        // No-op.
                        }
                    }
                });
                return;
            }
            if (type == EVT_CLIENT_NODE_DISCONNECTED || type == EVT_NODE_SEGMENTED || !ctx.clientDisconnected())
                discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, customMsg);
            if (stateFinishMsg != null)
                discoWrk.addEvent(EVT_DISCOVERY_CUSTOM_EVT, nextTopVer, node, discoCache, topSnapshot, stateFinishMsg);
        }
    });
    spi.setDataExchange(new DiscoverySpiDataExchange() {

        @Override
        public DiscoveryDataBag collect(DiscoveryDataBag dataBag) {
            assert dataBag != null;
            assert dataBag.joiningNodeId() != null;
            if (ctx.localNodeId().equals(dataBag.joiningNodeId())) {
                for (GridComponent c : ctx.components()) c.collectJoiningNodeData(dataBag);
            } else {
                for (GridComponent c : ctx.components()) c.collectGridNodeData(dataBag);
            }
            return dataBag;
        }

        @Override
        public void onExchange(DiscoveryDataBag dataBag) {
            assert dataBag != null;
            assert dataBag.joiningNodeId() != null;
            if (ctx.localNodeId().equals(dataBag.joiningNodeId())) {
                // NodeAdded msg reached joining node after round-trip over the ring.
                IGridClusterStateProcessor stateProc = ctx.state();
                stateProc.onGridDataReceived(dataBag.gridDiscoveryData(stateProc.discoveryDataType().ordinal()));
                for (GridComponent c : ctx.components()) {
                    if (c.discoveryDataType() != null && c != stateProc)
                        c.onGridDataReceived(dataBag.gridDiscoveryData(c.discoveryDataType().ordinal()));
                }
            } else {
                // Discovery data from newly joined node has to be applied to the current old node.
                IGridClusterStateProcessor stateProc = ctx.state();
                JoiningNodeDiscoveryData data0 = dataBag.newJoinerDiscoveryData(stateProc.discoveryDataType().ordinal());
                assert data0 != null;
                stateProc.onJoiningNodeDataReceived(data0);
                for (GridComponent c : ctx.components()) {
                    if (c.discoveryDataType() != null && c != stateProc) {
                        JoiningNodeDiscoveryData data = dataBag.newJoinerDiscoveryData(c.discoveryDataType().ordinal());
                        if (data != null)
                            c.onJoiningNodeDataReceived(data);
                    }
                }
            }
        }
    });
    startSpi();
    registeredDiscoSpi = true;
    try {
        U.await(startLatch);
    } catch (IgniteInterruptedException e) {
        throw new IgniteCheckedException("Failed to start discovery manager (thread has been interrupted).", e);
    }
    // Start segment check worker only if frequency is greater than 0.
    if (hasRslvrs && segChkFreq > 0) {
        segChkWrk = new SegmentCheckWorker();
        segChkThread = new IgniteThread(segChkWrk);
        segChkThread.start();
    }
    locNode = spi.getLocalNode();
    checkAttributes(discoCache().remoteNodes());
    // Start discovery worker.
    new IgniteThread(discoWrk).start();
    if (log.isDebugEnabled())
        log.debug(startInfo());
}

Example 3 with ChangeGlobalStateMessage

use of org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage in project ignite by apache.

the class GridDhtPartitionsExchangeFuture method init.

/**
 * Starts activity.
 *
 * @param newCrd {@code True} if node become coordinator on this exchange.
 * @throws IgniteInterruptedCheckedException If interrupted.
 */
public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
    if (isDone())
        return;
    assert !cctx.kernalContext().isDaemon();
    initTs = U.currentTimeMillis();
    U.await(evtLatch);
    assert firstDiscoEvt != null : this;
    assert exchId.nodeId().equals(firstDiscoEvt.eventNode().id()) : this;
    try {
        AffinityTopologyVersion topVer = initialVersion();
        srvNodes = new ArrayList<>(firstEvtDiscoCache.serverNodes());
        remaining.addAll(F.nodeIds(F.view(srvNodes, F.remoteNodes(cctx.localNodeId()))));
        crd = srvNodes.isEmpty() ? null : srvNodes.get(0);
        boolean crdNode = crd != null && crd.isLocal();
        exchCtx = new ExchangeContext(crdNode, this);
        assert state == null : state;
        if (crdNode)
            state = ExchangeLocalState.CRD;
        else
            state = cctx.kernalContext().clientNode() ? ExchangeLocalState.CLIENT : ExchangeLocalState.SRV;
        if (exchLog.isInfoEnabled()) {
            exchLog.info("Started exchange init [topVer=" + topVer + ", crd=" + crdNode + ", evt=" + IgniteUtils.gridEventName(firstDiscoEvt.type()) + ", evtNode=" + firstDiscoEvt.eventNode().id() + ", customEvt=" + (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT ? ((DiscoveryCustomEvent) firstDiscoEvt).customMessage() : null) + ", allowMerge=" + exchCtx.mergeExchanges() + ']');
        }
        ExchangeType exchange;
        if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
            assert !exchCtx.mergeExchanges();
            DiscoveryCustomMessage msg = ((DiscoveryCustomEvent) firstDiscoEvt).customMessage();
            forceAffReassignment = DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(msg) && firstEventCache().minimumNodeVersion().compareToIgnoreTimestamp(FORCE_AFF_REASSIGNMENT_SINCE) >= 0;
            if (msg instanceof ChangeGlobalStateMessage) {
                assert exchActions != null && !exchActions.empty();
                exchange = onClusterStateChangeRequest(crdNode);
            } else if (msg instanceof DynamicCacheChangeBatch) {
                assert exchActions != null && !exchActions.empty();
                exchange = onCacheChangeRequest(crdNode);
            } else if (msg instanceof SnapshotDiscoveryMessage) {
                exchange = onCustomMessageNoAffinityChange(crdNode);
            } else if (msg instanceof WalStateAbstractMessage)
                exchange = onCustomMessageNoAffinityChange(crdNode);
            else {
                assert affChangeMsg != null : this;
                exchange = onAffinityChangeRequest(crdNode);
            }
            if (forceAffReassignment)
                cctx.affinity().onCentralizedAffinityChange(this, crdNode);
            initCoordinatorCaches(newCrd);
        } else {
            if (firstDiscoEvt.type() == EVT_NODE_JOINED) {
                if (!firstDiscoEvt.eventNode().isLocal()) {
                    Collection<DynamicCacheDescriptor> receivedCaches = cctx.cache().startReceivedCaches(firstDiscoEvt.eventNode().id(), topVer);
                    cctx.affinity().initStartedCaches(crdNode, this, receivedCaches);
                } else
                    initCachesOnLocalJoin();
            }
            initCoordinatorCaches(newCrd);
            if (exchCtx.mergeExchanges()) {
                if (localJoinExchange()) {
                    if (cctx.kernalContext().clientNode()) {
                        onClientNodeEvent(crdNode);
                        exchange = ExchangeType.CLIENT;
                    } else {
                        onServerNodeEvent(crdNode);
                        exchange = ExchangeType.ALL;
                    }
                } else {
                    if (CU.clientNode(firstDiscoEvt.eventNode()))
                        exchange = onClientNodeEvent(crdNode);
                    else
                        exchange = cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
                }
                if (exchId.isLeft())
                    onLeft();
            } else {
                exchange = CU.clientNode(firstDiscoEvt.eventNode()) ? onClientNodeEvent(crdNode) : onServerNodeEvent(crdNode);
            }
        }
        updateTopologies(crdNode);
        switch(exchange) {
            case ALL:
                {
                    distributedExchange();
                    break;
                }
            case CLIENT:
                {
                    if (!exchCtx.mergeExchanges() && exchCtx.fetchAffinityOnJoin())
                        initTopologies();
                    clientOnlyExchange();
                    break;
                }
            case NONE:
                {
                    initTopologies();
                    onDone(topVer);
                    break;
                }
            default:
                assert false;
        }
        if (cctx.localNode().isClient())
            tryToPerformLocalSnapshotOperation();
        if (exchLog.isInfoEnabled())
            exchLog.info("Finished exchange init [topVer=" + topVer + ", crd=" + crdNode + ']');
    } catch (IgniteInterruptedCheckedException e) {
        onDone(e);
        throw e;
    } catch (IgniteNeedReconnectException e) {
        onDone(e);
    } catch (Throwable e) {
        if (reconnectOnError(e))
            onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
        else {
            U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
            onDone(e);
        }
        if (e instanceof Error)
            throw (Error) e;
    }
}