Examples with IoStatisticsHolder org.apache.ignite.internal.metric.IoStatisticsHolder used on opensource projects

Example 1 with IoStatisticsHolder

use of org.apache.ignite.internal.metric.IoStatisticsHolder in project ignite by apache.

the class GridCacheQueryManager method runQuery.

/**
 * Processes cache query request.
 *
 * @param qryInfo Query info.
 */
@SuppressWarnings("unchecked")
protected void runQuery(GridCacheQueryInfo qryInfo) {
    assert qryInfo != null;
    assert qryInfo.query().type() != SCAN || !qryInfo.local() : qryInfo;
    if (!enterBusy()) {
        if (cctx.localNodeId().equals(qryInfo.senderId()))
            throw new IllegalStateException("Failed to process query request (grid is stopping).");
        // Ignore remote requests when when node is stopping.
        return;
    }
    try {
        boolean performanceStatsEnabled = cctx.kernalContext().performanceStatistics().enabled();
        if (performanceStatsEnabled)
            IoStatisticsQueryHelper.startGatheringQueryStatistics();
        boolean loc = qryInfo.local();
        QueryResult<K, V> res = null;
        if (log.isDebugEnabled())
            log.debug("Running query: " + qryInfo);
        boolean rmvIter = true;
        GridCacheQueryAdapter<?> qry = qryInfo.query();
        try {
            // Preparing query closures.
            IgniteClosure<Cache.Entry<K, V>, Object> trans = (IgniteClosure<Cache.Entry<K, V>, Object>) qryInfo.transformer();
            IgniteReducer<Cache.Entry<K, V>, Object> rdc = (IgniteReducer<Cache.Entry<K, V>, Object>) qryInfo.reducer();
            injectResources(trans);
            injectResources(rdc);
            int pageSize = qry.pageSize();
            boolean incBackups = qry.includeBackups();
            String taskName = cctx.kernalContext().task().resolveTaskName(qry.taskHash());
            IgniteSpiCloseableIterator iter;
            GridCacheQueryType type;
            res = loc ? executeQuery(qry, qryInfo.arguments(), trans, loc, taskName, recipient(qryInfo.senderId(), qryInfo.requestId())) : queryResult(qryInfo, taskName);
            if (res == null)
                return;
            iter = res.iterator(recipient(qryInfo.senderId(), qryInfo.requestId()));
            type = res.type();
            final GridCacheAdapter<K, V> cache = cctx.cache();
            if (log.isDebugEnabled())
                log.debug("Received index iterator [iterHasNext=" + iter.hasNext() + ", cacheSize=" + cache.size() + ']');
            int cnt = 0;
            boolean stop = false;
            boolean pageSent = false;
            Collection<Object> data = new ArrayList<>(pageSize);
            AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion();
            final boolean statsEnabled = cctx.statisticsEnabled();
            final boolean readEvt = cctx.events().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
            CacheObjectContext objCtx = cctx.cacheObjectContext();
            while (!Thread.currentThread().isInterrupted()) {
                long start = statsEnabled ? System.nanoTime() : 0L;
                // actual row extracting may happen inside this method.
                if (!iter.hasNext())
                    break;
                Object row0 = iter.next();
                // Query is cancelled.
                if (row0 == null) {
                    onPageReady(loc, qryInfo, null, null, true, null);
                    break;
                }
                if (type == SCAN || type == INDEX)
                    // Scan iterator may return already transformed entry
                    data.add(row0);
                else {
                    IgniteBiTuple<K, V> row = (IgniteBiTuple<K, V>) row0;
                    final K key = row.getKey();
                    final V val = row.getValue();
                    if (log.isDebugEnabled()) {
                        ClusterNode primaryNode = cctx.affinity().primaryByKey(key, cctx.affinity().affinityTopologyVersion());
                        log.debug(S.toString("Record", "key", key, true, "val", val, true, "incBackups", incBackups, false, "priNode", primaryNode != null ? U.id8(primaryNode.id()) : null, false, "node", U.id8(cctx.localNode().id()), false));
                    }
                    if (val == null) {
                        if (log.isDebugEnabled())
                            log.debug(S.toString("Unsuitable record value", "val", val, true));
                        continue;
                    }
                    if (statsEnabled) {
                        CacheMetricsImpl metrics = cctx.cache().metrics0();
                        metrics.onRead(true);
                        metrics.addGetTimeNanos(System.nanoTime() - start);
                    }
                    K key0 = null;
                    V val0 = null;
                    if (readEvt && cctx.gridEvents().hasListener(EVT_CACHE_QUERY_OBJECT_READ)) {
                        key0 = (K) CacheObjectUtils.unwrapBinaryIfNeeded(objCtx, key, qry.keepBinary(), false, null);
                        val0 = (V) CacheObjectUtils.unwrapBinaryIfNeeded(objCtx, val, qry.keepBinary(), false, null);
                        switch(type) {
                            case SQL:
                                cctx.gridEvents().record(new CacheQueryReadEvent<>(cctx.localNode(), "SQL query entry read.", EVT_CACHE_QUERY_OBJECT_READ, CacheQueryType.SQL.name(), cctx.name(), qry.queryClassName(), qry.clause(), null, null, qryInfo.arguments(), securitySubjectId(cctx), taskName, key0, val0, null, null));
                                break;
                            case TEXT:
                                cctx.gridEvents().record(new CacheQueryReadEvent<>(cctx.localNode(), "Full text query entry read.", EVT_CACHE_QUERY_OBJECT_READ, CacheQueryType.FULL_TEXT.name(), cctx.name(), qry.queryClassName(), qry.clause(), null, null, null, securitySubjectId(cctx), taskName, key0, val0, null, null));
                                break;
                        }
                    }
                    if (rdc != null) {
                        if (key0 == null)
                            key0 = (K) CacheObjectUtils.unwrapBinaryIfNeeded(objCtx, key, qry.keepBinary(), false, null);
                        if (val0 == null)
                            val0 = (V) CacheObjectUtils.unwrapBinaryIfNeeded(objCtx, val, qry.keepBinary(), false, null);
                        Cache.Entry<K, V> entry = new CacheEntryImpl(key0, val0);
                        // Reduce.
                        if (!rdc.collect(entry) || !iter.hasNext()) {
                            onPageReady(loc, qryInfo, null, Collections.singletonList(rdc.reduce()), true, null);
                            pageSent = true;
                            break;
                        } else
                            continue;
                    } else {
                        if (type == TEXT)
                            // (K, V, score). Value transfers as BinaryObject.
                            data.add(row0);
                        else
                            data.add(new T2<>(key, val));
                    }
                }
                if (!loc) {
                    if (++cnt == pageSize || !iter.hasNext()) {
                        boolean finished = !iter.hasNext();
                        onPageReady(loc, qryInfo, res.metadata(), data, finished, null);
                        pageSent = true;
                        res.onPageSend();
                        if (!finished)
                            rmvIter = false;
                        if (!qryInfo.allPages())
                            return;
                        data = new ArrayList<>(pageSize);
                        if (stop)
                            // while
                            break;
                    }
                }
            }
            if (!pageSent) {
                if (rdc == null)
                    onPageReady(loc, qryInfo, res.metadata(), data, true, null);
                else
                    onPageReady(loc, qryInfo, res.metadata(), Collections.singletonList(rdc.reduce()), true, null);
                res.onPageSend();
            }
        } catch (Throwable e) {
            if (X.hasCause(e, ClassNotFoundException.class) && !qry.keepBinary() && cctx.binaryMarshaller() && !cctx.localNode().isClient() && !log.isQuiet()) {
                LT.warn(log, "Suggestion for the cause of ClassNotFoundException");
                LT.warn(log, "To disable, set -D" + IGNITE_QUIET + "=true");
                LT.warn(log, "  ^-- Ignite configured to use BinaryMarshaller but keepBinary is false for " + "request");
                LT.warn(log, "  ^-- Server node need to load definition of data classes. " + "It can be reason of ClassNotFoundException(consider IgniteCache.withKeepBinary to fix)");
                LT.warn(log, "Refer this page for detailed information: " + "https://apacheignite.readme.io/docs/binary-marshaller");
            }
            if (!X.hasCause(e, GridDhtUnreservedPartitionException.class))
                U.error(log, "Failed to run query [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e);
            onPageReady(loc, qryInfo, null, null, true, e);
            if (e instanceof Error)
                throw (Error) e;
        } finally {
            if (loc) {
                // Local iterators are always removed.
                if (res != null) {
                    try {
                        res.closeIfNotShared(recipient(qryInfo.senderId(), qryInfo.requestId()));
                    } catch (IgniteCheckedException e) {
                        if (!X.hasCause(e, GridDhtUnreservedPartitionException.class))
                            U.error(log, "Failed to close local iterator [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e);
                    }
                }
            } else if (rmvIter)
                removeQueryResult(qryInfo.senderId(), qryInfo.requestId());
            if (performanceStatsEnabled) {
                IoStatisticsHolder stat = IoStatisticsQueryHelper.finishGatheringQueryStatistics();
                if (stat.logicalReads() > 0 || stat.physicalReads() > 0) {
                    cctx.kernalContext().performanceStatistics().queryReads(res.type(), qryInfo.senderId(), qryInfo.requestId(), stat.logicalReads(), stat.physicalReads());
                }
            }
        }
    } finally {
        leaveBusy();
    }
}

Example 2 with IoStatisticsHolder

use of org.apache.ignite.internal.metric.IoStatisticsHolder in project ignite by apache.

the class CacheDataRowAdapter method initFromDataPage.

/**
 * @param io Data page IO.
 * @param pageAddr Data page address.
 * @param itemId Row item Id.
 * @param grp Cache group.
 * @param sharedCtx Cache shared context.
 * @param pageMem Page memory.
 * @param rowData Required row data.
 * @param skipVer Whether version read should be skipped.
 * @throws IgniteCheckedException If failed.
 */
public final void initFromDataPage(DataPageIO io, long pageAddr, int itemId, @Nullable CacheGroupContext grp, GridCacheSharedContext<?, ?> sharedCtx, PageMemory pageMem, RowData rowData, boolean skipVer) throws IgniteCheckedException {
    // Group is null if try evict page, with persistence evictions should be disabled.
    assert grp != null || pageMem instanceof PageMemoryNoStoreImpl;
    CacheObjectContext coctx = grp != null ? grp.cacheObjectContext() : null;
    boolean readCacheId = grp == null || grp.storeCacheIdInDataPage();
    int grpId = grp != null ? grp.groupId() : 0;
    IoStatisticsHolder statHolder = grp != null ? grp.statisticsHolderData() : IoStatisticsHolderNoOp.INSTANCE;
    IncompleteObject<?> incomplete = readIncomplete(null, sharedCtx, coctx, pageMem.pageSize(), pageMem.realPageSize(grpId), pageAddr, itemId, io, rowData, readCacheId, skipVer);
    if (incomplete != null) {
        // Initialize the remaining part of the large row from other pages.
        long nextLink = incomplete.getNextLink();
        if (nextLink != 0L) {
            doInitFromLink(nextLink, sharedCtx, coctx, pageMem, grpId, statHolder, readCacheId, rowData, incomplete, skipVer);
        }
    }
}

Example 3 with IoStatisticsHolder

use of org.apache.ignite.internal.metric.IoStatisticsHolder in project ignite by apache.

the class GridMapQueryExecutor method onQueryRequest0.

/**
 * @param node Node authored request.
 * @param qryId Query ID.
 * @param reqId Request ID.
 * @param segmentId index segment ID.
 * @param schemaName Schema name.
 * @param qrys Queries to execute.
 * @param cacheIds Caches which will be affected by these queries.
 * @param topVer Topology version.
 * @param partsMap Partitions map for unstable topology.
 * @param parts Explicit partitions for current node.
 * @param pageSize Page size.
 * @param distributedJoins Query distributed join mode.
 * @param enforceJoinOrder Enforce join order H2 flag.
 * @param replicated Replicated only flag.
 * @param timeout Query timeout.
 * @param params Query parameters.
 * @param lazy Streaming flag.
 * @param mvccSnapshot MVCC snapshot.
 * @param dataPageScanEnabled If data page scan is enabled.
 */
private void onQueryRequest0(final ClusterNode node, final long qryId, final long reqId, final int segmentId, final String schemaName, final Collection<GridCacheSqlQuery> qrys, final List<Integer> cacheIds, final AffinityTopologyVersion topVer, final Map<UUID, int[]> partsMap, final int[] parts, final int pageSize, final boolean distributedJoins, final boolean enforceJoinOrder, final boolean replicated, final int timeout, final Object[] params, boolean lazy, @Nullable final MvccSnapshot mvccSnapshot, Boolean dataPageScanEnabled, boolean treatReplicatedAsPartitioned) {
    boolean performanceStatsEnabled = ctx.performanceStatistics().enabled();
    if (performanceStatsEnabled)
        IoStatisticsQueryHelper.startGatheringQueryStatistics();
    // Prepare to run queries.
    GridCacheContext<?, ?> mainCctx = mainCacheContext(cacheIds);
    MapNodeResults nodeRess = resultsForNode(node.id());
    MapQueryResults qryResults = null;
    PartitionReservation reserved = null;
    QueryContext qctx = null;
    // We don't use try with resources on purpose - the catch block must also be executed in the context of this span.
    TraceSurroundings trace = MTC.support(ctx.tracing().create(SQL_QRY_EXEC_REQ, MTC.span()).addTag(SQL_QRY_TEXT, () -> qrys.stream().map(GridCacheSqlQuery::query).collect(Collectors.joining("; "))));
    try {
        if (topVer != null) {
            // Reserve primary for topology version or explicit partitions.
            reserved = h2.partitionReservationManager().reservePartitions(cacheIds, topVer, parts, node.id(), reqId);
            if (reserved.failed()) {
                sendRetry(node, reqId, segmentId, reserved.error());
                return;
            }
        }
        // Prepare query context.
        DistributedJoinContext distributedJoinCtx = null;
        if (distributedJoins && !replicated) {
            distributedJoinCtx = new DistributedJoinContext(topVer, partsMap, node.id(), reqId, segmentId, pageSize);
        }
        qctx = new QueryContext(segmentId, h2.backupFilter(topVer, parts, treatReplicatedAsPartitioned), distributedJoinCtx, mvccSnapshot, reserved, true);
        qryResults = new MapQueryResults(h2, reqId, qrys.size(), mainCctx, lazy, qctx);
        // qctx is set, we have to release reservations inside of it.
        reserved = null;
        if (distributedJoinCtx != null)
            qryCtxRegistry.setShared(node.id(), reqId, qctx);
        if (nodeRess.put(reqId, segmentId, qryResults) != null)
            throw new IllegalStateException();
        if (nodeRess.cancelled(reqId)) {
            qryCtxRegistry.clearShared(node.id(), reqId);
            nodeRess.cancelRequest(reqId);
            throw new QueryCancelledException();
        }
        // Run queries.
        int qryIdx = 0;
        boolean evt = mainCctx != null && mainCctx.events().isRecordable(EVT_CACHE_QUERY_EXECUTED);
        for (GridCacheSqlQuery qry : qrys) {
            H2PooledConnection conn = h2.connections().connection(schemaName);
            H2Utils.setupConnection(conn, qctx, distributedJoins, enforceJoinOrder, lazy);
            MapQueryResult res = new MapQueryResult(h2, mainCctx, node.id(), qry, params, conn, log);
            qryResults.addResult(qryIdx, res);
            try {
                res.lock();
                // Ensure we are on the target node for this replicated query.
                if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) {
                    String sql = qry.query();
                    Collection<Object> params0 = F.asList(qry.parameters(params));
                    PreparedStatement stmt = conn.prepareStatement(sql, H2StatementCache.queryFlags(distributedJoins, enforceJoinOrder));
                    H2Utils.bindParameters(stmt, params0);
                    MapH2QueryInfo qryInfo = new MapH2QueryInfo(stmt, qry.query(), node.id(), qryId, reqId, segmentId);
                    ResultSet rs = h2.executeSqlQueryWithTimer(stmt, conn, sql, timeout, qryResults.queryCancel(qryIdx), dataPageScanEnabled, qryInfo);
                    if (evt) {
                        ctx.event().record(new CacheQueryExecutedEvent<>(node, "SQL query executed.", EVT_CACHE_QUERY_EXECUTED, CacheQueryType.SQL.name(), mainCctx.name(), null, qry.query(), null, null, params, node.id(), null));
                    }
                    assert rs instanceof JdbcResultSet : rs.getClass();
                    if (qryResults.cancelled()) {
                        rs.close();
                        throw new QueryCancelledException();
                    }
                    res.openResult(rs, qryInfo);
                    final GridQueryNextPageResponse msg = prepareNextPage(nodeRess, node, qryResults, qryIdx, segmentId, pageSize, dataPageScanEnabled);
                    if (msg != null)
                        sendNextPage(node, msg);
                } else {
                    assert !qry.isPartitioned();
                    qryResults.closeResult(qryIdx);
                }
                qryIdx++;
            } finally {
                try {
                    res.unlockTables();
                } finally {
                    res.unlock();
                }
            }
        }
        if (!lazy)
            qryResults.releaseQueryContext();
    } catch (Throwable e) {
        if (qryResults != null) {
            nodeRess.remove(reqId, segmentId, qryResults);
            qryResults.close();
            // If a query is cancelled before execution is started partitions have to be released.
            if (!lazy || !qryResults.isAllClosed())
                qryResults.releaseQueryContext();
        } else
            releaseReservations(qctx);
        if (e instanceof QueryCancelledException)
            sendError(node, reqId, e);
        else {
            SQLException sqlEx = X.cause(e, SQLException.class);
            if (sqlEx != null && sqlEx.getErrorCode() == ErrorCode.STATEMENT_WAS_CANCELED)
                sendQueryCancel(node, reqId);
            else {
                GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class);
                if (retryErr != null) {
                    final String retryCause = String.format("Failed to execute non-collocated query (will retry) [localNodeId=%s, rmtNodeId=%s, reqId=%s, " + "errMsg=%s]", ctx.localNodeId(), node.id(), reqId, retryErr.getMessage());
                    sendRetry(node, reqId, segmentId, retryCause);
                } else {
                    QueryRetryException qryRetryErr = X.cause(e, QueryRetryException.class);
                    if (qryRetryErr != null)
                        sendError(node, reqId, qryRetryErr);
                    else {
                        if (e instanceof Error) {
                            U.error(log, "Failed to execute local query.", e);
                            throw (Error) e;
                        }
                        U.warn(log, "Failed to execute local query.", e);
                        sendError(node, reqId, e);
                    }
                }
            }
        }
    } finally {
        if (reserved != null)
            reserved.release();
        if (trace != null)
            trace.close();
        if (performanceStatsEnabled) {
            IoStatisticsHolder stat = IoStatisticsQueryHelper.finishGatheringQueryStatistics();
            if (stat.logicalReads() > 0 || stat.physicalReads() > 0) {
                ctx.performanceStatistics().queryReads(GridCacheQueryType.SQL_FIELDS, node.id(), reqId, stat.logicalReads(), stat.physicalReads());
            }
        }
    }
}

Example 4 with IoStatisticsHolder

use of org.apache.ignite.internal.metric.IoStatisticsHolder in project ignite by apache.

the class GridCacheDistributedQueryManager method scanQueryDistributed.

/**
 * {@inheritDoc}
 */
@SuppressWarnings({ "unchecked" })
@Override
public GridCloseableIterator scanQueryDistributed(final GridCacheQueryAdapter qry, Collection<ClusterNode> nodes) throws IgniteCheckedException {
    assert !cctx.isLocal() : cctx.name();
    assert qry.type() == GridCacheQueryType.SCAN : qry;
    assert qry.mvccSnapshot() != null || !cctx.mvccEnabled();
    boolean performanceStatsEnabled = cctx.kernalContext().performanceStatistics().enabled();
    long startTime = performanceStatsEnabled ? System.currentTimeMillis() : 0;
    long startTimeNanos = performanceStatsEnabled ? System.nanoTime() : 0;
    GridCloseableIterator locIter0 = null;
    for (ClusterNode node : nodes) {
        if (node.isLocal()) {
            locIter0 = scanQueryLocal(qry, false);
            Collection<ClusterNode> rmtNodes = new ArrayList<>(nodes.size() - 1);
            for (ClusterNode n : nodes) {
                // Equals by reference can be used here.
                if (n != node)
                    rmtNodes.add(n);
            }
            nodes = rmtNodes;
            break;
        }
    }
    final GridCloseableIterator locIter = locIter0;
    final GridCacheQueryBean bean = new GridCacheQueryBean(qry, null, qry.<K, V>transform(), null);
    final CacheQueryFuture fut = queryDistributed(bean, nodes);
    return new GridCloseableIteratorAdapter() {

        /**
         */
        private Object cur;

        /**
         * Logical reads.
         */
        private long logicalReads;

        /**
         * Physical reads.
         */
        private long physicalReads;

        @Override
        protected Object onNext() throws IgniteCheckedException {
            if (!onHasNext())
                throw new NoSuchElementException();
            Object e = cur;
            cur = null;
            return e;
        }

        @Override
        protected boolean onHasNext() throws IgniteCheckedException {
            if (cur != null)
                return true;
            if (locIter != null) {
                if (performanceStatsEnabled)
                    IoStatisticsQueryHelper.startGatheringQueryStatistics();
                try {
                    if (locIter.hasNextX())
                        cur = locIter.nextX();
                } finally {
                    if (performanceStatsEnabled) {
                        IoStatisticsHolder stat = IoStatisticsQueryHelper.finishGatheringQueryStatistics();
                        logicalReads += stat.logicalReads();
                        physicalReads += stat.physicalReads();
                    }
                }
            }
            return cur != null || (cur = convert(fut.next())) != null;
        }

        /**
         * @param obj Entry to convert.
         * @return Cache entry
         */
        private Object convert(Object obj) {
            if (qry.transform() != null)
                return obj;
            Map.Entry e = (Map.Entry) obj;
            return e == null ? null : new CacheQueryEntry(e.getKey(), e.getValue());
        }

        @Override
        protected void onClose() throws IgniteCheckedException {
            super.onClose();
            if (locIter != null)
                locIter.close();
            if (fut != null)
                fut.cancel();
            if (performanceStatsEnabled) {
                cctx.kernalContext().performanceStatistics().query(SCAN, cctx.name(), ((GridCacheDistributedQueryFuture) fut).requestId(), startTime, System.nanoTime() - startTimeNanos, true);
                if (logicalReads > 0 || physicalReads > 0) {
                    cctx.kernalContext().performanceStatistics().queryReads(SCAN, cctx.localNodeId(), ((GridCacheDistributedQueryFuture) fut).requestId(), logicalReads, physicalReads);
                }
            }
        }
    };
}

Example 5 with IoStatisticsHolder

use of org.apache.ignite.internal.metric.IoStatisticsHolder in project ignite by apache.

the class CacheDataRowAdapter method initFromLink.

/**
 * Read row from data pages.
 * Can be called with cctx == null, if cache instance is unknown, but its ID is stored in the data row.
 *
 * @param grp Cache group.
 * @param sharedCtx Shared context.
 * @param pageMem Page memory.
 * @param rowData Row data.
 * @param skipVer Whether version read should be skipped.
 * @throws IgniteCheckedException If failed.
 */
public final void initFromLink(@Nullable CacheGroupContext grp, GridCacheSharedContext<?, ?> sharedCtx, PageMemory pageMem, RowData rowData, boolean skipVer) throws IgniteCheckedException {
    // Group is null if try evict page, with persistence evictions should be disabled.
    assert grp != null || pageMem instanceof PageMemoryNoStoreImpl;
    CacheObjectContext coctx = grp != null ? grp.cacheObjectContext() : null;
    boolean readCacheId = grp == null || grp.storeCacheIdInDataPage();
    int grpId = grp != null ? grp.groupId() : 0;
    IoStatisticsHolder statHolder = grp != null ? grp.statisticsHolderData() : IoStatisticsHolderNoOp.INSTANCE;
    doInitFromLink(link, sharedCtx, coctx, pageMem, grpId, statHolder, readCacheId, rowData, null, skipVer);
}