use of com.baidu.hugegraph.backend.page.IdHolder.BatchIdHolder in project incubator-hugegraph by apache.
the class GraphIndexTransaction method doJointIndex.
@Watched(prefix = "index")
private IdHolder doJointIndex(IndexQueries queries) {
if (queries.oomRisk()) {
LOG.warn("There is OOM risk if the joint operation is based on a " + "large amount of data, please use single index + filter " + "instead of joint index: {}", queries.rootQuery());
}
// All queries are joined with AND
Set<Id> intersectIds = null;
boolean filtering = false;
IdHolder resultHolder = null;
for (Map.Entry<IndexLabel, ConditionQuery> e : queries.entrySet()) {
IndexLabel indexLabel = e.getKey();
ConditionQuery query = e.getValue();
assert !query.paging();
if (!query.noLimit() && queries.size() > 1) {
// Unset limit for intersection operation
query.limit(Query.NO_LIMIT);
}
/*
* Try to query by joint indexes:
* 1 If there is any index exceeded the threshold, transform into
* partial index query, then filter after back-table.
* 1.1 Return the holder of the first index that not exceeded the
* threshold if there exists one index, this holder will be used
* as the only query condition.
* 1.2 Return the holder of the first index if all indexes exceeded
* the threshold.
* 2 Else intersect holders for all indexes, and return intersection
* ids of all indexes.
*/
IdHolder holder = this.doIndexQuery(indexLabel, query);
if (resultHolder == null) {
resultHolder = holder;
this.storeSelectedIndexField(indexLabel, query);
}
// default value is 1000
assert this.indexIntersectThresh > 0;
Set<Id> ids = ((BatchIdHolder) holder).peekNext(this.indexIntersectThresh).ids();
if (ids.size() >= this.indexIntersectThresh) {
// Transform into filtering
filtering = true;
query.optimized(OptimizedType.INDEX_FILTER);
} else if (filtering) {
assert ids.size() < this.indexIntersectThresh;
resultHolder = holder;
this.storeSelectedIndexField(indexLabel, query);
break;
} else {
if (intersectIds == null) {
intersectIds = ids;
} else {
CollectionUtil.intersectWithModify(intersectIds, ids);
}
if (intersectIds.isEmpty()) {
break;
}
}
}
if (filtering) {
return resultHolder;
} else {
assert intersectIds != null;
return new FixedIdHolder(queries.asJointQuery(), intersectIds);
}
}
use of com.baidu.hugegraph.backend.page.IdHolder.BatchIdHolder in project incubator-hugegraph by apache.
the class GraphIndexTransaction method doIndexQueryBatch.
@Watched(prefix = "index")
private IdHolder doIndexQueryBatch(IndexLabel indexLabel, ConditionQuery query) {
Iterator<BackendEntry> entries = super.query(query).iterator();
return new BatchIdHolder(query, entries, batch -> {
LockUtil.Locks locks = new LockUtil.Locks(this.graphName());
try {
// Catch lock every batch
locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id());
locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id());
if (!indexLabel.system()) {
/*
* Check exist because it may be deleted after some batches
* throw exception if the index label not exists
* NOTE: graph() will return null with system index label
*/
graph().indexLabel(indexLabel.id());
}
// Iterate one batch, and keep iterator position
Set<Id> ids = InsertionOrderUtil.newSet();
while ((batch == Query.NO_LIMIT || ids.size() < batch) && entries.hasNext()) {
HugeIndex index = this.serializer.readIndex(graph(), query, entries.next());
this.removeExpiredIndexIfNeeded(index, query.showExpired());
ids.addAll(index.elementIds());
Query.checkForceCapacity(ids.size());
this.recordIndexValue(query, index);
}
return ids;
} finally {
locks.unlock();
}
});
}
Aggregations