Examples with IgniteReducer org.apache.ignite.lang.IgniteReducer used on opensource projects

Example 1 with IgniteReducer

use of org.apache.ignite.lang.IgniteReducer in project ignite by apache.

the class GridCacheQueryManager method runQuery.

/**
     * Processes cache query request.
     *
     * @param qryInfo Query info.
     */
@SuppressWarnings("unchecked")
protected void runQuery(GridCacheQueryInfo qryInfo) {
    assert qryInfo != null;
    assert qryInfo.query().type() != SCAN || !qryInfo.local() : qryInfo;
    if (!enterBusy()) {
        if (cctx.localNodeId().equals(qryInfo.senderId()))
            throw new IllegalStateException("Failed to process query request (grid is stopping).");
        // Ignore remote requests when when node is stopping.
        return;
    }
    try {
        boolean loc = qryInfo.local();
        QueryResult<K, V> res = null;
        if (log.isDebugEnabled())
            log.debug("Running query: " + qryInfo);
        boolean rmvIter = true;
        try {
            // Preparing query closures.
            IgniteClosure<Cache.Entry<K, V>, Object> trans = (IgniteClosure<Cache.Entry<K, V>, Object>) qryInfo.transformer();
            IgniteReducer<Cache.Entry<K, V>, Object> rdc = (IgniteReducer<Cache.Entry<K, V>, Object>) qryInfo.reducer();
            injectResources(trans);
            injectResources(rdc);
            GridCacheQueryAdapter<?> qry = qryInfo.query();
            int pageSize = qry.pageSize();
            boolean incBackups = qry.includeBackups();
            String taskName = cctx.kernalContext().task().resolveTaskName(qry.taskHash());
            IgniteSpiCloseableIterator<IgniteBiTuple<K, V>> iter;
            GridCacheQueryType type;
            res = loc ? executeQuery(qry, qryInfo.arguments(), loc, qry.subjectId(), taskName, recipient(qryInfo.senderId(), qryInfo.requestId())) : queryResult(qryInfo, taskName);
            if (res == null)
                return;
            iter = res.iterator(recipient(qryInfo.senderId(), qryInfo.requestId()));
            type = res.type();
            final GridCacheAdapter<K, V> cache = cctx.cache();
            if (log.isDebugEnabled())
                log.debug("Received index iterator [iterHasNext=" + iter.hasNext() + ", cacheSize=" + cache.size() + ']');
            int cnt = 0;
            boolean stop = false;
            boolean pageSent = false;
            Collection<Object> data = new ArrayList<>(pageSize);
            AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion();
            final boolean statsEnabled = cctx.config().isStatisticsEnabled();
            final boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
            while (!Thread.currentThread().isInterrupted() && iter.hasNext()) {
                long start = statsEnabled ? System.nanoTime() : 0L;
                IgniteBiTuple<K, V> row = iter.next();
                // Query is cancelled.
                if (row == null) {
                    onPageReady(loc, qryInfo, null, true, null);
                    break;
                }
                final K key = row.getKey();
                // Other types are filtered in indexing manager.
                if (!cctx.isReplicated() && qry.type() == SCAN && qry.partition() == null && cctx.config().getCacheMode() != LOCAL && !incBackups && !cctx.affinity().primaryByKey(cctx.localNode(), key, topVer)) {
                    if (log.isDebugEnabled())
                        log.debug("Ignoring backup element [row=" + row + ", cacheMode=" + cctx.config().getCacheMode() + ", incBackups=" + incBackups + ", primary=" + cctx.affinity().primaryByKey(cctx.localNode(), key, topVer) + ']');
                    continue;
                }
                V val = row.getValue();
                if (log.isDebugEnabled()) {
                    ClusterNode primaryNode = cctx.affinity().primaryByKey(key, cctx.affinity().affinityTopologyVersion());
                    log.debug(S.toString("Record", "key", key, true, "val", val, true, "incBackups", incBackups, false, "priNode", primaryNode != null ? U.id8(primaryNode.id()) : null, false, "node", U.id8(cctx.localNode().id()), false));
                }
                if (val == null) {
                    if (log.isDebugEnabled())
                        log.debug(S.toString("Unsuitable record value", "val", val, true));
                    continue;
                }
                if (statsEnabled) {
                    CacheMetricsImpl metrics = cctx.cache().metrics0();
                    metrics.onRead(true);
                    metrics.addGetTimeNanos(System.nanoTime() - start);
                }
                K key0 = null;
                V val0 = null;
                if (readEvt) {
                    key0 = (K) cctx.unwrapBinaryIfNeeded(key, qry.keepBinary());
                    val0 = (V) cctx.unwrapBinaryIfNeeded(val, qry.keepBinary());
                    switch(type) {
                        case SQL:
                            cctx.gridEvents().record(new CacheQueryReadEvent<>(cctx.localNode(), "SQL query entry read.", EVT_CACHE_QUERY_OBJECT_READ, CacheQueryType.SQL.name(), cctx.name(), qry.queryClassName(), qry.clause(), null, null, qryInfo.arguments(), qry.subjectId(), taskName, key0, val0, null, null));
                            break;
                        case TEXT:
                            cctx.gridEvents().record(new CacheQueryReadEvent<>(cctx.localNode(), "Full text query entry read.", EVT_CACHE_QUERY_OBJECT_READ, CacheQueryType.FULL_TEXT.name(), cctx.name(), qry.queryClassName(), qry.clause(), null, null, null, qry.subjectId(), taskName, key0, val0, null, null));
                            break;
                        case SCAN:
                            cctx.gridEvents().record(new CacheQueryReadEvent<>(cctx.localNode(), "Scan query entry read.", EVT_CACHE_QUERY_OBJECT_READ, CacheQueryType.SCAN.name(), cctx.name(), null, null, qry.scanFilter(), null, null, qry.subjectId(), taskName, key0, val0, null, null));
                            break;
                    }
                }
                if (rdc != null || trans != null) {
                    if (key0 == null)
                        key0 = (K) cctx.unwrapBinaryIfNeeded(key, qry.keepBinary());
                    if (val0 == null)
                        val0 = (V) cctx.unwrapBinaryIfNeeded(val, qry.keepBinary());
                    Cache.Entry<K, V> entry = new CacheEntryImpl(key0, val0);
                    // Reduce.
                    if (rdc != null) {
                        if (!rdc.collect(entry) || !iter.hasNext()) {
                            onPageReady(loc, qryInfo, Collections.singletonList(rdc.reduce()), true, null);
                            pageSent = true;
                            break;
                        } else
                            continue;
                    }
                    data.add(trans != null ? trans.apply(entry) : !loc ? new GridCacheQueryResponseEntry<>(key, val) : F.t(key, val));
                } else
                    data.add(!loc ? new GridCacheQueryResponseEntry<>(key, val) : F.t(key, val));
                if (!loc) {
                    if (++cnt == pageSize || !iter.hasNext()) {
                        boolean finished = !iter.hasNext();
                        onPageReady(loc, qryInfo, data, finished, null);
                        pageSent = true;
                        if (!finished)
                            rmvIter = false;
                        if (!qryInfo.allPages())
                            return;
                        data = new ArrayList<>(pageSize);
                        if (stop)
                            // while
                            break;
                    }
                }
            }
            if (!pageSent) {
                if (rdc == null)
                    onPageReady(loc, qryInfo, data, true, null);
                else
                    onPageReady(loc, qryInfo, Collections.singletonList(rdc.reduce()), true, null);
            }
        } catch (Throwable e) {
            if (!X.hasCause(e, GridDhtUnreservedPartitionException.class))
                U.error(log, "Failed to run query [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e);
            onPageReady(loc, qryInfo, null, true, e);
            if (e instanceof Error)
                throw (Error) e;
        } finally {
            if (loc) {
                // Local iterators are always removed.
                if (res != null) {
                    try {
                        res.closeIfNotShared(recipient(qryInfo.senderId(), qryInfo.requestId()));
                    } catch (IgniteCheckedException e) {
                        if (!X.hasCause(e, GridDhtUnreservedPartitionException.class))
                            U.error(log, "Failed to close local iterator [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e);
                    }
                }
            } else if (rmvIter)
                removeQueryResult(qryInfo.senderId(), qryInfo.requestId());
        }
    } finally {
        leaveBusy();
    }
}

Example 2 with IgniteReducer

use of org.apache.ignite.lang.IgniteReducer in project ignite by apache.

the class GridClosureProcessorSelfTest method testReducerError.

/**
     * @throws Exception If failed.
     */
public void testReducerError() throws Exception {
    final Ignite g = grid(0);
    final Collection<IgniteCallable<Integer>> jobs = new ArrayList<>();
    for (int i = 0; i < g.cluster().nodes().size(); i++) {
        jobs.add(new IgniteCallable<Integer>() {

            @Override
            public Integer call() throws Exception {
                throw new RuntimeException("Test exception.");
            }
        });
    }
    GridTestUtils.assertThrows(log, new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            g.compute().call(jobs, new IgniteReducer<Integer, Object>() {

                @Override
                public boolean collect(@Nullable Integer e) {
                    fail("Expects failed jobs never call 'collect' method.");
                    return true;
                }

                @Override
                public Object reduce() {
                    return null;
                }
            });
            return null;
        }
    }, IgniteException.class, null);
}

Example 3 with IgniteReducer

use of org.apache.ignite.lang.IgniteReducer in project ignite by apache.

the class GridCacheQueryManager method runFieldsQuery.

/**
     * Processes fields query request.
     *
     * @param qryInfo Query info.
     */
protected void runFieldsQuery(GridCacheQueryInfo qryInfo) {
    assert qryInfo != null;
    if (!enterBusy()) {
        if (cctx.localNodeId().equals(qryInfo.senderId()))
            throw new IllegalStateException("Failed to process query request (grid is stopping).");
        // Ignore remote requests when when node is stopping.
        return;
    }
    try {
        if (log.isDebugEnabled())
            log.debug("Running query: " + qryInfo);
        boolean rmvRes = true;
        FieldsResult res = null;
        final boolean statsEnabled = cctx.config().isStatisticsEnabled();
        final boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
        try {
            // Preparing query closures.
            IgniteReducer<Object, Object> rdc = (IgniteReducer<Object, Object>) qryInfo.reducer();
            injectResources(rdc);
            GridCacheQueryAdapter<?> qry = qryInfo.query();
            int pageSize = qry.pageSize();
            Collection<Object> data = null;
            Collection<Object> entities = null;
            if (qryInfo.local() || rdc != null || cctx.isLocalNode(qryInfo.senderId()))
                data = new ArrayList<>(pageSize);
            else
                entities = new ArrayList<>(pageSize);
            String taskName = cctx.kernalContext().task().resolveTaskName(qry.taskHash());
            res = qryInfo.local() ? executeFieldsQuery(qry, qryInfo.arguments(), qryInfo.local(), qry.subjectId(), taskName, recipient(qryInfo.senderId(), qryInfo.requestId())) : fieldsQueryResult(qryInfo, taskName);
            // If metadata needs to be returned to user and cleaned from internal fields - copy it.
            List<GridQueryFieldMetadata> meta = qryInfo.includeMetaData() ? (res.metaData() != null ? new ArrayList<>(res.metaData()) : null) : res.metaData();
            if (!qryInfo.includeMetaData())
                meta = null;
            GridCloseableIterator<?> it = new GridSpiCloseableIteratorWrapper<Object>(res.iterator(recipient(qryInfo.senderId(), qryInfo.requestId())));
            if (log.isDebugEnabled())
                log.debug("Received fields iterator [iterHasNext=" + it.hasNext() + ']');
            if (!it.hasNext()) {
                if (rdc != null)
                    data = Collections.singletonList(rdc.reduce());
                onFieldsPageReady(qryInfo.local(), qryInfo, meta, entities, data, true, null);
                return;
            }
            int cnt = 0;
            boolean metaSent = false;
            while (!Thread.currentThread().isInterrupted() && it.hasNext()) {
                long start = statsEnabled ? System.nanoTime() : 0L;
                Object row = it.next();
                // Query is cancelled.
                if (row == null) {
                    onPageReady(qryInfo.local(), qryInfo, null, true, null);
                    break;
                }
                if (statsEnabled) {
                    CacheMetricsImpl metrics = cctx.cache().metrics0();
                    metrics.onRead(true);
                    metrics.addGetTimeNanos(System.nanoTime() - start);
                }
                if (readEvt) {
                    cctx.gridEvents().record(new CacheQueryReadEvent<K, V>(cctx.localNode(), "SQL fields query result set row read.", EVT_CACHE_QUERY_OBJECT_READ, CacheQueryType.SQL_FIELDS.name(), cctx.name(), null, qry.clause(), null, null, qryInfo.arguments(), qry.subjectId(), taskName, null, null, null, row));
                }
                if ((qryInfo.local() || rdc != null || cctx.isLocalNode(qryInfo.senderId()))) {
                    // Reduce.
                    if (rdc != null) {
                        if (!rdc.collect(row))
                            break;
                    } else
                        data.add(row);
                } else
                    entities.add(row);
                if (rdc == null && ((!qryInfo.allPages() && ++cnt == pageSize) || !it.hasNext())) {
                    onFieldsPageReady(qryInfo.local(), qryInfo, !metaSent ? meta : null, entities, data, !it.hasNext(), null);
                    if (it.hasNext())
                        rmvRes = false;
                    if (!qryInfo.allPages())
                        return;
                }
            }
            if (rdc != null) {
                onFieldsPageReady(qryInfo.local(), qryInfo, meta, null, Collections.singletonList(rdc.reduce()), true, null);
            }
        } catch (IgniteCheckedException e) {
            if (log.isDebugEnabled() || !e.hasCause(SQLException.class))
                U.error(log, "Failed to run fields query [qry=" + qryInfo + ", node=" + cctx.nodeId() + ']', e);
            else {
                if (e.hasCause(SQLException.class))
                    U.error(log, "Failed to run fields query [node=" + cctx.nodeId() + ", msg=" + e.getCause(SQLException.class).getMessage() + ']');
                else
                    U.error(log, "Failed to run fields query [node=" + cctx.nodeId() + ", msg=" + e.getMessage() + ']');
            }
            onFieldsPageReady(qryInfo.local(), qryInfo, null, null, null, true, e);
        } catch (Throwable e) {
            U.error(log, "Failed to run fields query [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e);
            onFieldsPageReady(qryInfo.local(), qryInfo, null, null, null, true, e);
            if (e instanceof Error)
                throw (Error) e;
        } finally {
            if (qryInfo.local()) {
                // Don't we need to always remove local iterators?
                if (rmvRes && res != null) {
                    try {
                        res.closeIfNotShared(recipient(qryInfo.senderId(), qryInfo.requestId()));
                    } catch (IgniteCheckedException e) {
                        U.error(log, "Failed to close local iterator [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e);
                    }
                }
            } else if (rmvRes)
                removeFieldsQueryResult(qryInfo.senderId(), qryInfo.requestId());
        }
    } finally {
        leaveBusy();
    }
}

Example 4 with IgniteReducer

use of org.apache.ignite.lang.IgniteReducer in project ignite by apache.

the class GridCacheQueryRequest method beforeLocalExecution.

/**
     * @param ctx Context.
     * @throws IgniteCheckedException In case of error.
     */
void beforeLocalExecution(GridCacheContext ctx) throws IgniteCheckedException {
    Marshaller marsh = ctx.marshaller();
    rdc = rdc != null ? U.<IgniteReducer<Object, Object>>unmarshal(marsh, U.marshal(marsh, rdc), U.resolveClassLoader(ctx.gridConfig())) : null;
    trans = trans != null ? U.<IgniteClosure<Object, Object>>unmarshal(marsh, U.marshal(marsh, trans), U.resolveClassLoader(ctx.gridConfig())) : null;
}