Examples with PriorityTracker org.opensearch.ad.caching.PriorityTracker used on opensource projects

Example 1 with PriorityTracker

use of org.opensearch.ad.caching.PriorityTracker in project anomaly-detection by opensearch-project.

the class ADBatchTaskRunner method getTopEntities.

/**
 * Get top entities for HC detector. Will use similar logic of realtime detector,
 * but split the whole historical detection date range into limited number of
 * buckets (1000 buckets by default). Will get top entities for each bucket, then
 * put each bucket's top entities into {@link PriorityTracker} to track top
 * entities dynamically. Once all buckets done, we can get finalized top entities
 * in {@link PriorityTracker}.
 *
 * @param adTask AD task
 * @param internalHCListener internal HC listener
 */
public void getTopEntities(ADTask adTask, ActionListener<String> internalHCListener) {
    getDateRangeOfSourceData(adTask, (dataStartTime, dataEndTime) -> {
        PriorityTracker priorityTracker = new PriorityTracker(Clock.systemUTC(), adTask.getDetector().getDetectorIntervalInSeconds(), adTask.getDetectionDateRange().getStartTime().toEpochMilli(), MAX_TOP_ENTITIES_LIMIT_FOR_HISTORICAL_ANALYSIS);
        long detectorInterval = adTask.getDetector().getDetectorIntervalInMilliseconds();
        logger.debug("start to search top entities at {}, data start time: {}, data end time: {}, interval: {}", System.currentTimeMillis(), dataStartTime, dataEndTime, detectorInterval);
        if (adTask.getDetector().isMultiCategoryDetector()) {
            searchTopEntitiesForMultiCategoryHC(adTask, priorityTracker, dataEndTime, Math.max((dataEndTime - dataStartTime) / MAX_TOP_ENTITY_SEARCH_BUCKETS, detectorInterval), dataStartTime, dataStartTime + detectorInterval, internalHCListener);
        } else {
            searchTopEntitiesForSingleCategoryHC(adTask, priorityTracker, dataEndTime, Math.max((dataEndTime - dataStartTime) / MAX_TOP_ENTITY_SEARCH_BUCKETS, detectorInterval), dataStartTime, dataStartTime + detectorInterval, internalHCListener);
        }
    }, internalHCListener);
}

Example 2 with PriorityTracker

use of org.opensearch.ad.caching.PriorityTracker in project anomaly-detection by opensearch-project.

the class ADBatchTaskRunner method searchTopEntitiesForSingleCategoryHC.

private void searchTopEntitiesForSingleCategoryHC(ADTask adTask, PriorityTracker priorityTracker, long detectionEndTime, long interval, long dataStartTime, long dataEndTime, ActionListener<String> internalHCListener) {
    checkIfADTaskCancelledAndCleanupCache(adTask);
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
    RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(adTask.getDetector().getTimeField()).gte(dataStartTime).lte(dataEndTime).format("epoch_millis");
    boolQueryBuilder.filter(rangeQueryBuilder);
    boolQueryBuilder.filter(adTask.getDetector().getFilterQuery());
    sourceBuilder.query(boolQueryBuilder);
    String topEntitiesAgg = "topEntities";
    AggregationBuilder aggregation = new TermsAggregationBuilder(topEntitiesAgg).field(adTask.getDetector().getCategoryField().get(0)).size(MAX_TOP_ENTITIES_LIMIT_FOR_HISTORICAL_ANALYSIS);
    sourceBuilder.aggregation(aggregation).size(0);
    SearchRequest searchRequest = new SearchRequest();
    searchRequest.source(sourceBuilder);
    searchRequest.indices(adTask.getDetector().getIndices().toArray(new String[0]));
    client.search(searchRequest, ActionListener.wrap(r -> {
        StringTerms stringTerms = r.getAggregations().get(topEntitiesAgg);
        List<StringTerms.Bucket> buckets = stringTerms.getBuckets();
        List<String> topEntities = new ArrayList<>();
        for (StringTerms.Bucket bucket : buckets) {
            String key = bucket.getKeyAsString();
            topEntities.add(key);
        }
        topEntities.forEach(e -> priorityTracker.updatePriority(e));
        if (dataEndTime < detectionEndTime) {
            searchTopEntitiesForSingleCategoryHC(adTask, priorityTracker, detectionEndTime, interval, dataEndTime, dataEndTime + interval, internalHCListener);
        } else {
            logger.debug("finish searching top entities at " + System.currentTimeMillis());
            List<String> topNEntities = priorityTracker.getTopNEntities(maxTopEntitiesPerHcDetector);
            if (topNEntities.size() == 0) {
                logger.error("There is no entity found for detector " + adTask.getDetectorId());
                internalHCListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "No entity found"));
                return;
            }
            adTaskCacheManager.addPendingEntities(adTask.getDetectorId(), topNEntities);
            adTaskCacheManager.setTopEntityCount(adTask.getDetectorId(), topNEntities.size());
            internalHCListener.onResponse("Get top entities done");
        }
    }, e -> {
        logger.error("Failed to get top entities for detector " + adTask.getDetectorId(), e);
        internalHCListener.onFailure(e);
    }));
}