Examples with Watched com.baidu.hugegraph.perf.PerfUtil.Watched used on opensource projects

Example 21 with Watched

use of com.baidu.hugegraph.perf.PerfUtil.Watched in project incubator-hugegraph by apache.

the class GraphIndexTransaction method doSearchIndex.

@Watched(prefix = "index")
private IdHolderList doSearchIndex(ConditionQuery query, MatchedIndex index) {
    query = this.constructSearchQuery(query, index);
    // Sorted by matched count
    IdHolderList holders = new SortByCountIdHolderList(query.paging());
    List<ConditionQuery> flatten = ConditionQueryFlatten.flatten(query);
    for (ConditionQuery q : flatten) {
        if (!q.noLimit() && flatten.size() > 1) {
            // Increase limit for union operation
            increaseLimit(q);
        }
        IndexQueries queries = index.constructIndexQueries(q);
        assert !query.paging() || queries.size() <= 1;
        IdHolder holder = this.doSingleOrJointIndex(queries);
        // NOTE: ids will be merged into one IdHolder if not in paging
        holders.add(holder);
    }
    return holders;
}

Example 22 with Watched

use of com.baidu.hugegraph.perf.PerfUtil.Watched in project incubator-hugegraph by apache.

the class GraphIndexTransaction method queryByUserprop.

@Watched(prefix = "index")
private IdHolderList queryByUserprop(ConditionQuery query) {
    // related index labels
    if (!this.graph().readMode().showOlap()) {
        for (Id pkId : query.userpropKeys()) {
            PropertyKey propertyKey = this.graph().propertyKey(pkId);
            if (propertyKey.olap()) {
                throw new NotAllowException("Not allowed to query by olap property key '%s'" + " when graph-read-mode is '%s'", propertyKey, this.graph().readMode());
            }
        }
    }
    Set<MatchedIndex> indexes = this.collectMatchedIndexes(query);
    if (indexes.isEmpty()) {
        Id label = query.condition(HugeKeys.LABEL);
        throw noIndexException(this.graph(), query, label);
    }
    // Value type of Condition not matched
    boolean paging = query.paging();
    if (!validQueryConditionValues(this.graph(), query)) {
        return IdHolderList.empty(paging);
    }
    // Do index query
    IdHolderList holders = new IdHolderList(paging);
    for (MatchedIndex index : indexes) {
        for (IndexLabel il : index.indexLabels()) {
            validateIndexLabel(il);
        }
        if (paging && index.indexLabels().size() > 1) {
            throw new NotSupportException("joint index query in paging");
        }
        if (index.containsSearchIndex()) {
            // Do search-index query
            holders.addAll(this.doSearchIndex(query, index));
        } else {
            // Do secondary-index, range-index or shard-index query
            IndexQueries queries = index.constructIndexQueries(query);
            assert !paging || queries.size() <= 1;
            IdHolder holder = this.doSingleOrJointIndex(queries);
            holders.add(holder);
        }
    /*
             * NOTE: need to skip the offset if offset > 0, but can't handle
             * it here because the query may a sub-query after flatten,
             * so the offset will be handle in QueryList.IndexQuery
             *
             * TODO: finish early here if records exceeds required limit with
             *       FixedIdHolder.
             */
    }
    return holders;
}

Example 23 with Watched

use of com.baidu.hugegraph.perf.PerfUtil.Watched in project incubator-hugegraph by apache.

the class AbstractTransaction method query.

@Watched(prefix = "tx")
public QueryResults<BackendEntry> query(Query query) {
    LOG.debug("Transaction query: {}", query);
    /*
         * NOTE: it's dangerous if an IdQuery/ConditionQuery is empty
         * check if the query is empty and its class is not the Query itself
         */
    if (query.empty() && !query.getClass().equals(Query.class)) {
        throw new BackendException("Query without any id or condition");
    }
    Query squery = this.serializer.writeQuery(query);
    // Do rate limit if needed
    RateLimiter rateLimiter = this.graph.readRateLimiter();
    if (rateLimiter != null && query.resultType().isGraph()) {
        double time = rateLimiter.acquire(1);
        if (time > 0) {
            LOG.debug("Waited for {}s to query", time);
        }
        BackendEntryIterator.checkInterrupted();
    }
    this.beforeRead();
    try {
        this.injectOlapPkIfNeeded(squery);
        return new QueryResults<>(this.store.query(squery), query);
    } finally {
        // TODO: not complete the iteration currently
        this.afterRead();
    }
}

Example 24 with Watched

use of com.baidu.hugegraph.perf.PerfUtil.Watched in project incubator-hugegraph by apache.

the class GraphIndexTransaction method doJointIndex.

@Watched(prefix = "index")
private IdHolder doJointIndex(IndexQueries queries) {
    if (queries.oomRisk()) {
        LOG.warn("There is OOM risk if the joint operation is based on a " + "large amount of data, please use single index + filter " + "instead of joint index: {}", queries.rootQuery());
    }
    // All queries are joined with AND
    Set<Id> intersectIds = null;
    boolean filtering = false;
    IdHolder resultHolder = null;
    for (Map.Entry<IndexLabel, ConditionQuery> e : queries.entrySet()) {
        IndexLabel indexLabel = e.getKey();
        ConditionQuery query = e.getValue();
        assert !query.paging();
        if (!query.noLimit() && queries.size() > 1) {
            // Unset limit for intersection operation
            query.limit(Query.NO_LIMIT);
        }
        /*
             * Try to query by joint indexes:
             * 1 If there is any index exceeded the threshold, transform into
             *   partial index query, then filter after back-table.
             * 1.1 Return the holder of the first index that not exceeded the
             *     threshold if there exists one index, this holder will be used
             *     as the only query condition.
             * 1.2 Return the holder of the first index if all indexes exceeded
             *     the threshold.
             * 2 Else intersect holders for all indexes, and return intersection
             *   ids of all indexes.
             */
        IdHolder holder = this.doIndexQuery(indexLabel, query);
        if (resultHolder == null) {
            resultHolder = holder;
            this.storeSelectedIndexField(indexLabel, query);
        }
        // default value is 1000
        assert this.indexIntersectThresh > 0;
        Set<Id> ids = ((BatchIdHolder) holder).peekNext(this.indexIntersectThresh).ids();
        if (ids.size() >= this.indexIntersectThresh) {
            // Transform into filtering
            filtering = true;
            query.optimized(OptimizedType.INDEX_FILTER);
        } else if (filtering) {
            assert ids.size() < this.indexIntersectThresh;
            resultHolder = holder;
            this.storeSelectedIndexField(indexLabel, query);
            break;
        } else {
            if (intersectIds == null) {
                intersectIds = ids;
            } else {
                CollectionUtil.intersectWithModify(intersectIds, ids);
            }
            if (intersectIds.isEmpty()) {
                break;
            }
        }
    }
    if (filtering) {
        return resultHolder;
    } else {
        assert intersectIds != null;
        return new FixedIdHolder(queries.asJointQuery(), intersectIds);
    }
}

Example 25 with Watched

use of com.baidu.hugegraph.perf.PerfUtil.Watched in project incubator-hugegraph by apache.

the class GraphIndexTransaction method doIndexQueryBatch.

@Watched(prefix = "index")
private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) {
    Iterator<BackendEntry> entries = super.query(query).iterator();
    return new BatchIdHolder(query, entries, batch -> {
        LockUtil.Locks locks = new LockUtil.Locks(this.graphName());
        try {
            // Catch lock every batch
            locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id());
            locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id());
            if (!indexLabel.system()) {
                /*
                     * Check exist because it may be deleted after some batches
                     * throw exception if the index label not exists
                     * NOTE: graph() will return null with system index label
                     */
                graph().indexLabel(indexLabel.id());
            }
            // Iterate one batch, and keep iterator position
            Set<Id> ids = InsertionOrderUtil.newSet();
            while ((batch == Query.NO_LIMIT || ids.size() < batch) && entries.hasNext()) {
                HugeIndex index = this.serializer.readIndex(graph(), query, entries.next());
                this.removeExpiredIndexIfNeeded(index, query.showExpired());
                ids.addAll(index.elementIds());
                Query.checkForceCapacity(ids.size());
                this.recordIndexValue(query, index);
            }
            return ids;
        } finally {
            locks.unlock();
        }
    });
}