Examples with ReducePartitionMapResult org.apache.ignite.internal.processors.query.h2.twostep.ReducePartitionMapResult used on opensource projects

Example 1 with ReducePartitionMapResult

use of org.apache.ignite.internal.processors.query.h2.twostep.ReducePartitionMapResult in project gridgain by gridgain.

the class GridReduceQueryExecutor method update.

/**
 * @param schemaName Schema name.
 * @param cacheIds Cache ids.
 * @param selectQry Select query.
 * @param params SQL parameters.
 * @param enforceJoinOrder Enforce join order of tables.
 * @param pageSize Page size.
 * @param timeoutMillis Timeout.
 * @param parts Partitions.
 * @param isReplicatedOnly Whether query uses only replicated caches.
 * @param cancel Cancel state.
 * @return Update result, or {@code null} when some map node doesn't support distributed DML.
 */
@SuppressWarnings("IfMayBeConditional")
public UpdateResult update(String schemaName, List<Integer> cacheIds, String selectQry, Object[] params, boolean enforceJoinOrder, int pageSize, int timeoutMillis, final int[] parts, boolean isReplicatedOnly, GridQueryCancel cancel) {
    AffinityTopologyVersion topVer = h2.readyTopologyVersion();
    ReducePartitionMapResult nodesParts = mapper.nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly);
    Collection<ClusterNode> nodes = nodesParts.nodes();
    if (F.isEmpty(nodes))
        throw new CacheException("Failed to determine nodes participating in the update. " + "Explanation (Retry update once topology recovers).");
    if (isReplicatedOnly) {
        ClusterNode locNode = ctx.discovery().localNode();
        if (nodes.contains(locNode))
            nodes = singletonList(locNode);
        else
            nodes = singletonList(F.rand(nodes));
    }
    for (ClusterNode n : nodes) {
        if (!n.version().greaterThanEqual(2, 3, 0)) {
            log.warning("Server-side DML optimization is skipped because map node does not support it. " + "Falling back to normal DML. [node=" + n.id() + ", v=" + n.version() + "].");
            return null;
        }
    }
    final long reqId = qryReqIdGen.incrementAndGet();
    final DmlDistributedUpdateRun r = new DmlDistributedUpdateRun(nodes.size());
    int flags = enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0;
    if (isReplicatedOnly)
        flags |= GridH2QueryRequest.FLAG_REPLICATED;
    GridH2DmlRequest req = new GridH2DmlRequest().requestId(reqId).topologyVersion(topVer).caches(cacheIds).schemaName(schemaName).query(selectQry).pageSize(pageSize).parameters(params).timeout(timeoutMillis).explicitTimeout(true).flags(flags);
    updRuns.put(reqId, r);
    boolean release = false;
    try {
        Map<ClusterNode, IntArray> partsMap = (nodesParts.queryPartitionsMap() != null) ? nodesParts.queryPartitionsMap() : nodesParts.partitionsMap();
        ReducePartitionsSpecializer partsSpec = (parts == null) ? null : new ReducePartitionsSpecializer(partsMap);
        final Collection<ClusterNode> finalNodes = nodes;
        cancel.add(() -> {
            r.future().onCancelled();
            send(finalNodes, new GridQueryCancelRequest(reqId), null, true);
        });
        // send() logs the debug message
        if (send(nodes, req, partsSpec, false))
            return r.future().get();
        throw new CacheException("Failed to send update request to participating nodes.");
    } catch (IgniteCheckedException | RuntimeException e) {
        release = true;
        U.error(log, "Error during update [localNodeId=" + ctx.localNodeId() + "]", e);
        throw new CacheException("Failed to run SQL update query. " + e.getMessage(), e);
    } finally {
        if (release)
            send(nodes, new GridQueryCancelRequest(reqId), null, false);
        if (!updRuns.remove(reqId, r))
            U.warn(log, "Update run was already removed: " + reqId);
    }
}

Example 2 with ReducePartitionMapResult

use of org.apache.ignite.internal.processors.query.h2.twostep.ReducePartitionMapResult in project ignite by apache.

the class GridReduceQueryExecutor method update.

/**
 * @param schemaName Schema name.
 * @param cacheIds Cache ids.
 * @param selectQry Select query.
 * @param params SQL parameters.
 * @param enforceJoinOrder Enforce join order of tables.
 * @param pageSize Page size.
 * @param timeoutMillis Timeout.
 * @param parts Partitions.
 * @param isReplicatedOnly Whether query uses only replicated caches.
 * @param cancel Cancel state.
 * @return Update result, or {@code null} when some map node doesn't support distributed DML.
 */
@SuppressWarnings("IfMayBeConditional")
public UpdateResult update(String schemaName, List<Integer> cacheIds, String selectQry, Object[] params, boolean enforceJoinOrder, int pageSize, int timeoutMillis, final int[] parts, boolean isReplicatedOnly, GridQueryCancel cancel) {
    AffinityTopologyVersion topVer = h2.readyTopologyVersion();
    ReducePartitionMapResult nodesParts = mapper.nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly);
    Collection<ClusterNode> nodes = nodesParts.nodes();
    if (F.isEmpty(nodes))
        throw new CacheException("Failed to determine nodes participating in the update. " + "Explanation (Retry update once topology recovers).");
    if (isReplicatedOnly) {
        ClusterNode locNode = ctx.discovery().localNode();
        if (nodes.contains(locNode))
            nodes = singletonList(locNode);
        else
            nodes = singletonList(F.rand(nodes));
    }
    for (ClusterNode n : nodes) {
        if (!n.version().greaterThanEqual(2, 3, 0)) {
            log.warning("Server-side DML optimization is skipped because map node does not support it. " + "Falling back to normal DML. [node=" + n.id() + ", v=" + n.version() + "].");
            return null;
        }
    }
    final long reqId = qryReqIdGen.incrementAndGet();
    h2.runningQueryManager().trackRequestId(reqId);
    final DmlDistributedUpdateRun r = new DmlDistributedUpdateRun(nodes.size());
    int flags = enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0;
    if (isReplicatedOnly)
        flags |= GridH2QueryRequest.FLAG_REPLICATED;
    GridH2DmlRequest req = new GridH2DmlRequest().requestId(reqId).topologyVersion(topVer).caches(cacheIds).schemaName(schemaName).query(selectQry).pageSize(pageSize).parameters(params).timeout(timeoutMillis).explicitTimeout(true).flags(flags);
    updRuns.put(reqId, r);
    boolean release = false;
    try {
        Map<ClusterNode, IntArray> partsMap = (nodesParts.queryPartitionsMap() != null) ? nodesParts.queryPartitionsMap() : nodesParts.partitionsMap();
        ReducePartitionsSpecializer partsSpec = (parts == null) ? null : new ReducePartitionsSpecializer(partsMap);
        final Collection<ClusterNode> finalNodes = nodes;
        cancel.add(() -> {
            r.future().onCancelled();
            send(finalNodes, new GridQueryCancelRequest(reqId), null, true);
        });
        // send() logs the debug message
        if (send(nodes, req, partsSpec, false))
            return r.future().get();
        throw new CacheException("Failed to send update request to participating nodes.");
    } catch (IgniteCheckedException | RuntimeException e) {
        release = true;
        U.error(log, "Error during update [localNodeId=" + ctx.localNodeId() + "]", e);
        throw new CacheException("Failed to run SQL update query. " + e.getMessage(), e);
    } finally {
        if (release)
            send(nodes, new GridQueryCancelRequest(reqId), null, false);
        if (!updRuns.remove(reqId, r))
            U.warn(log, "Update run was already removed: " + reqId);
    }
}

Example 3 with ReducePartitionMapResult

use of org.apache.ignite.internal.processors.query.h2.twostep.ReducePartitionMapResult in project ignite by apache.

the class GridReduceQueryExecutor method query.

/**
 * @param qryId Query ID.
 * @param schemaName Schema name.
 * @param qry Query.
 * @param keepBinary Keep binary.
 * @param enforceJoinOrder Enforce join order of tables.
 * @param timeoutMillis Timeout in milliseconds.
 * @param cancel Query cancel.
 * @param params Query parameters.
 * @param parts Partitions.
 * @param lazy Lazy execution flag.
 * @param mvccTracker Query tracker.
 * @param dataPageScanEnabled If data page scan is enabled.
 * @param pageSize Page size.
 * @return Rows iterator.
 */
@SuppressWarnings("IfMayBeConditional")
public Iterator<List<?>> query(long qryId, String schemaName, final GridCacheTwoStepQuery qry, boolean keepBinary, boolean enforceJoinOrder, int timeoutMillis, GridQueryCancel cancel, Object[] params, int[] parts, boolean lazy, MvccQueryTracker mvccTracker, Boolean dataPageScanEnabled, int pageSize) {
    assert !qry.mvccEnabled() || mvccTracker != null;
    if (pageSize <= 0)
        pageSize = Query.DFLT_PAGE_SIZE;
    // If explicit partitions are set, but there are no real tables, ignore.
    if (!qry.hasCacheIds() && parts != null)
        parts = null;
    // Partitions are not supported for queries over all replicated caches.
    if (parts != null && qry.isReplicatedOnly())
        throw new CacheException("Partitions are not supported for replicated caches");
    try {
        if (qry.mvccEnabled())
            checkActive(tx(ctx));
    } catch (IgniteTxAlreadyCompletedCheckedException e) {
        throw new TransactionAlreadyCompletedException(e.getMessage(), e);
    }
    final boolean singlePartMode = parts != null && parts.length == 1;
    if (F.isEmpty(params))
        params = EMPTY_PARAMS;
    List<Integer> cacheIds = qry.cacheIds();
    List<GridCacheSqlQuery> mapQueries = prepareMapQueries(qry, params, singlePartMode);
    final boolean skipMergeTbl = !qry.explain() && qry.skipMergeTable() || singlePartMode;
    final long retryTimeout = retryTimeout(timeoutMillis);
    final long qryStartTime = U.currentTimeMillis();
    ReduceQueryRun lastRun = null;
    for (int attempt = 0; ; attempt++) {
        ensureQueryNotCancelled(cancel);
        if (attempt > 0) {
            throttleOnRetry(lastRun, qryStartTime, retryTimeout, attempt);
            ensureQueryNotCancelled(cancel);
        }
        AffinityTopologyVersion topVer = h2.readyTopologyVersion();
        // Check if topology has changed while retrying on locked topology.
        if (h2.serverTopologyChanged(topVer) && ctx.cache().context().lockedTopologyVersion(null) != null) {
            throw new CacheException(new TransactionException("Server topology is changed during query " + "execution inside a transaction. It's recommended to rollback and retry transaction."));
        }
        ReducePartitionMapResult mapping = createMapping(qry, parts, cacheIds, topVer);
        if (// Can't map query.
        mapping == null)
            // Retry.
            continue;
        final Collection<ClusterNode> nodes = mapping.nodes();
        final Map<ClusterNode, Integer> nodeToSegmentsCnt = createNodeToSegmentsCountMapping(qry, mapping);
        assert !F.isEmpty(nodes);
        H2PooledConnection conn = h2.connections().connection(schemaName);
        final long qryReqId = qryReqIdGen.incrementAndGet();
        h2.runningQueryManager().trackRequestId(qryReqId);
        boolean release = true;
        try {
            final ReduceQueryRun r = createReduceQueryRun(conn, mapQueries, nodes, pageSize, nodeToSegmentsCnt, skipMergeTbl, qry.explain(), dataPageScanEnabled);
            runs.put(qryReqId, r);
            try {
                cancel.add(() -> send(nodes, new GridQueryCancelRequest(qryReqId), null, true));
                GridH2QueryRequest req = new GridH2QueryRequest().queryId(qryId).requestId(qryReqId).topologyVersion(topVer).pageSize(pageSize).caches(qry.cacheIds()).tables(qry.distributedJoins() ? qry.tables() : null).partitions(convert(mapping.partitionsMap())).queries(mapQueries).parameters(params).flags(queryFlags(qry, enforceJoinOrder, lazy, dataPageScanEnabled)).timeout(timeoutMillis).explicitTimeout(true).schemaName(schemaName);
                if (mvccTracker != null)
                    req.mvccSnapshot(mvccTracker.snapshot());
                final C2<ClusterNode, Message, Message> spec = parts == null ? null : new ReducePartitionsSpecializer(mapping.queryPartitionsMap());
                boolean retry = false;
                if (send(nodes, req, spec, false)) {
                    awaitAllReplies(r, nodes, cancel);
                    if (r.hasErrorOrRetry()) {
                        CacheException err = r.exception();
                        if (err != null) {
                            if (err.getCause() instanceof IgniteClientDisconnectedException)
                                throw err;
                            else if (QueryUtils.wasCancelled(err))
                                // Throw correct exception.
                                throw new QueryCancelledException();
                            throw err;
                        }
                        // If remote node asks us to retry then we have outdated full partition map.
                        h2.awaitForReadyTopologyVersion(r.retryTopologyVersion());
                        retry = true;
                    }
                } else
                    retry = true;
                if (retry) {
                    lastRun = runs.get(qryReqId);
                    assert lastRun != null;
                    // Retry.
                    continue;
                }
                Iterator<List<?>> resIter;
                if (skipMergeTbl) {
                    resIter = new ReduceIndexIterator(this, nodes, r, qryReqId, qry.distributedJoins(), mvccTracker, ctx.tracing());
                    release = false;
                    U.close(conn, log);
                } else {
                    ensureQueryNotCancelled(cancel);
                    QueryContext qctx = new QueryContext(0, null, null, null, null, true);
                    H2Utils.setupConnection(conn, qctx, false, enforceJoinOrder);
                    if (qry.explain())
                        return explainPlan(conn, qry, params);
                    GridCacheSqlQuery rdc = qry.reduceQuery();
                    final PreparedStatement stmt = conn.prepareStatementNoCache(rdc.query());
                    H2Utils.bindParameters(stmt, F.asList(rdc.parameters(params)));
                    ReduceH2QueryInfo qryInfo = new ReduceH2QueryInfo(stmt, qry.originalSql(), ctx.localNodeId(), qryId, qryReqId);
                    ResultSet res = h2.executeSqlQueryWithTimer(stmt, conn, rdc.query(), timeoutMillis, cancel, dataPageScanEnabled, qryInfo);
                    resIter = new H2FieldsIterator(res, mvccTracker, conn, r.pageSize(), log, h2, qryInfo, ctx.tracing());
                    conn = null;
                    // To prevent callback inside finally block;
                    mvccTracker = null;
                }
                return new GridQueryCacheObjectsIterator(resIter, h2.objectContext(), keepBinary);
            } catch (IgniteCheckedException | RuntimeException e) {
                release = true;
                if (e instanceof CacheException) {
                    if (QueryUtils.wasCancelled(e))
                        throw new CacheException("Failed to run reduce query locally.", new QueryCancelledException());
                    throw (CacheException) e;
                }
                Throwable cause = e;
                if (e instanceof IgniteCheckedException) {
                    Throwable disconnectedErr = ((IgniteCheckedException) e).getCause(IgniteClientDisconnectedException.class);
                    if (disconnectedErr != null)
                        cause = disconnectedErr;
                }
                throw new CacheException("Failed to run reduce query locally. " + cause.getMessage(), cause);
            } finally {
                if (release) {
                    releaseRemoteResources(nodes, r, qryReqId, qry.distributedJoins(), mvccTracker);
                    if (!skipMergeTbl) {
                        for (int i = 0, mapQrys = mapQueries.size(); i < mapQrys; i++) // Drop all merge tables.
                        fakeTable(null, i).innerTable(null);
                    }
                }
            }
        } finally {
            if (conn != null && release)
                U.close(conn, log);
        }
    }
}

Example 4 with ReducePartitionMapResult

use of org.apache.ignite.internal.processors.query.h2.twostep.ReducePartitionMapResult in project gridgain by gridgain.

the class GridReduceQueryExecutor method query.

/**
 * @param schemaName Schema name.
 * @param qry Query.
 * @param keepBinary Keep binary.
 * @param enforceJoinOrder Enforce join order of tables.
 * @param timeoutMillis Timeout in milliseconds.
 * @param cancel Query cancel.
 * @param params Query parameters.
 * @param parts Partitions.
 * @param lazy Lazy execution flag.
 * @param mvccTracker Query tracker.
 * @param dataPageScanEnabled If data page scan is enabled.
 * @param pageSize Page size.
 * @param maxMem Query memory limit.
 * @return Rows iterator.
 */
public Iterator<List<?>> query(@Nullable final Long qryId, String schemaName, final GridCacheTwoStepQuery qry, boolean keepBinary, boolean enforceJoinOrder, int timeoutMillis, GridQueryCancel cancel, Object[] params, int[] parts, boolean lazy, MvccQueryTracker mvccTracker, Boolean dataPageScanEnabled, int pageSize, long maxMem) {
    assert !qry.mvccEnabled() || mvccTracker != null;
    if (pageSize <= 0)
        pageSize = Query.DFLT_PAGE_SIZE;
    // If explicit partitions are set, but there are no real tables, ignore.
    if (!qry.hasCacheIds() && parts != null)
        parts = null;
    // Partitions are not supported for queries over all replicated caches.
    if (parts != null && qry.isReplicatedOnly())
        throw new CacheException("Partitions are not supported for replicated caches");
    try {
        if (qry.mvccEnabled())
            checkActive(tx(ctx));
    } catch (IgniteTxAlreadyCompletedCheckedException e) {
        throw new TransactionAlreadyCompletedException(e.getMessage(), e);
    }
    final boolean singlePartMode = parts != null && parts.length == 1;
    if (F.isEmpty(params))
        params = EMPTY_PARAMS;
    List<Integer> cacheIds = qry.cacheIds();
    List<GridCacheSqlQuery> mapQueries = prepareMapQueries(qry, params, singlePartMode);
    final boolean skipMergeTbl = !qry.explain() && qry.skipMergeTable() || singlePartMode;
    final long retryTimeout = retryTimeout(timeoutMillis);
    final long qryStartTime = U.currentTimeMillis();
    ReduceQueryRun lastRun = null;
    for (int attempt = 0; ; attempt++) {
        ensureQueryNotCancelled(cancel);
        if (attempt > 0) {
            throttleOnRetry(lastRun, qryStartTime, retryTimeout, attempt);
            ensureQueryNotCancelled(cancel);
        }
        AffinityTopologyVersion topVer = h2.readyTopologyVersion();
        // Check if topology has changed while retrying on locked topology.
        if (h2.serverTopologyChanged(topVer) && ctx.cache().context().lockedTopologyVersion(null) != null) {
            throw new CacheException(new TransactionException("Server topology is changed during query " + "execution inside a transaction. It's recommended to rollback and retry transaction."));
        }
        ReducePartitionMapResult mapping = createMapping(qry, parts, cacheIds, topVer);
        if (// Can't map query.
        mapping == null)
            // Retry.
            continue;
        final Collection<ClusterNode> nodes = mapping.nodes();
        final Map<ClusterNode, Integer> nodeToSegmentsCnt = createNodeToSegmentsCountMapping(qry, mapping);
        assert !F.isEmpty(nodes);
        H2PooledConnection conn = h2.connections().connection(schemaName);
        final long qryReqId = qryReqIdGen.incrementAndGet();
        boolean release = true;
        try {
            final ReduceQueryRun r = createReduceQueryRun(conn, mapQueries, nodes, pageSize, nodeToSegmentsCnt, skipMergeTbl, qry.explain(), dataPageScanEnabled);
            runs.put(qryReqId, r);
            try {
                cancel.add(() -> send(nodes, new GridQueryCancelRequest(qryReqId), null, true));
                GridH2QueryRequest req = new GridH2QueryRequest().requestId(qryReqId).topologyVersion(topVer).pageSize(pageSize).caches(qry.cacheIds()).tables(qry.distributedJoins() ? qry.tables() : null).partitions(convert(mapping.partitionsMap())).queries(mapQueries).parameters(params).flags(queryFlags(qry, enforceJoinOrder, lazy, dataPageScanEnabled)).timeout(timeoutMillis).explicitTimeout(true).schemaName(schemaName).maxMemory(maxMem).runningQryId(qryId);
                if (mvccTracker != null)
                    req.mvccSnapshot(mvccTracker.snapshot());
                final C2<ClusterNode, Message, Message> spec = parts == null ? null : new ReducePartitionsSpecializer(mapping.queryPartitionsMap());
                boolean retry = false;
                if (send(nodes, req, spec, false)) {
                    awaitAllReplies(r, nodes, cancel);
                    if (r.hasErrorOrRetry()) {
                        CacheException err = r.exception();
                        if (err != null) {
                            if (err.getCause() instanceof IgniteClientDisconnectedException)
                                throw err;
                            else if (QueryUtils.wasCancelled(err))
                                // Throw correct exception.
                                throw new QueryCancelledException();
                            throw err;
                        }
                        // If remote node asks us to retry then we have outdated full partition map.
                        h2.awaitForReadyTopologyVersion(r.retryTopologyVersion());
                        retry = true;
                    }
                } else
                    retry = true;
                if (retry) {
                    lastRun = runs.get(qryReqId);
                    assert lastRun != null;
                    // Retry.
                    continue;
                }
                Iterator<List<?>> resIter;
                if (skipMergeTbl) {
                    resIter = new ReduceIndexIterator(this, nodes, r, qryReqId, qry.distributedJoins(), mvccTracker, ctx.tracing());
                    release = false;
                    U.close(conn, log);
                } else {
                    ensureQueryNotCancelled(cancel);
                    QueryContext qctx = new QueryContext(0, null, null, null, null, true);
                    H2Utils.setupConnection(conn, qctx, false, enforceJoinOrder);
                    if (qry.explain())
                        return explainPlan(conn, qry, params);
                    GridCacheSqlQuery rdc = qry.reduceQuery();
                    final PreparedStatement stmt = conn.prepareStatementNoCache(rdc.query());
                    H2Utils.bindParameters(stmt, F.asList(rdc.parameters(params)));
                    ReduceH2QueryInfo qryInfo = new ReduceH2QueryInfo(stmt, qry.originalSql(), ctx.discovery().localNode(), qryReqId, qryId);
                    r.reducers().forEach(reducer -> reducer.memoryTracker(h2.memTracker(qryInfo)));
                    ResultSet res = h2.executeSqlQueryWithTimer(stmt, conn, rdc.query(), timeoutMillis, cancel, dataPageScanEnabled, qryInfo, maxMem);
                    resIter = new H2FieldsIterator(res, mvccTracker, conn, r.pageSize(), log, h2, qryInfo, ctx.tracing());
                    conn = null;
                    // To prevent callback inside finally block;
                    mvccTracker = null;
                }
                return new GridQueryCacheObjectsIterator(resIter, h2.objectContext(), keepBinary);
            } catch (IgniteCheckedException | RuntimeException e) {
                release = true;
                if (e instanceof CacheException) {
                    if (QueryUtils.wasCancelled(e))
                        throw new CacheException("Failed to run reduce query locally.", new QueryCancelledException());
                    throw (CacheException) e;
                }
                Throwable cause = e;
                if (e instanceof IgniteCheckedException) {
                    Throwable disconnectedErr = ((IgniteCheckedException) e).getCause(IgniteClientDisconnectedException.class);
                    if (disconnectedErr != null)
                        cause = disconnectedErr;
                }
                throw new CacheException("Failed to run reduce query locally. " + cause.getMessage(), cause);
            } finally {
                if (release) {
                    releaseRemoteResources(nodes, r, qryReqId, qry.distributedJoins(), mvccTracker);
                    if (!skipMergeTbl) {
                        for (int i = 0, mapQrys = mapQueries.size(); i < mapQrys; i++) // Drop all merge tables.
                        fakeTable(null, i).innerTable(null);
                    }
                }
            }
        } finally {
            if (conn != null && release)
                U.close(conn, log);
        }
    }
}