Search in sources :

Example 1 with PARENT_TASK_ID_FIELD

use of org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD in project anomaly-detection by opensearch-project.

the class ADTaskManager method resetEntityTasksAsStopped.

private void resetEntityTasksAsStopped(String detectorTaskId) {
    UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest();
    updateByQueryRequest.indices(DETECTION_STATE_INDEX);
    BoolQueryBuilder query = new BoolQueryBuilder();
    query.filter(new TermQueryBuilder(PARENT_TASK_ID_FIELD, detectorTaskId));
    query.filter(new TermQueryBuilder(TASK_TYPE_FIELD, ADTaskType.HISTORICAL_HC_ENTITY.name()));
    query.filter(new TermsQueryBuilder(STATE_FIELD, NOT_ENDED_STATES));
    updateByQueryRequest.setQuery(query);
    updateByQueryRequest.setRefresh(true);
    String script = String.format(Locale.ROOT, "ctx._source.%s='%s';", STATE_FIELD, ADTaskState.STOPPED.name());
    updateByQueryRequest.setScript(new Script(script));
    client.execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(r -> {
        List<BulkItemResponse.Failure> bulkFailures = r.getBulkFailures();
        if (isNullOrEmpty(bulkFailures)) {
            logger.debug("Updated {} child entity tasks state for detector task {}", r.getUpdated(), detectorTaskId);
        } else {
            logger.error("Failed to update child entity task's state for detector task {} ", detectorTaskId);
        }
    }, e -> logger.error("Exception happened when update child entity task's state for detector task " + detectorTaskId, e)));
}
Also used : PARENT_TASK_ID_FIELD(org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) BulkAction(org.opensearch.action.bulk.BulkAction) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ExceptionUtil.getShardsFailure(org.opensearch.ad.util.ExceptionUtil.getShardsFailure) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) STOPPED_BY_FIELD(org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD) WriteRequest(org.opensearch.action.support.WriteRequest) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) ForwardADTaskAction(org.opensearch.ad.transport.ForwardADTaskAction) ActionListener(org.opensearch.action.ActionListener) DETECTOR_IS_RUNNING(org.opensearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) UpdateByQueryAction(org.opensearch.index.reindex.UpdateByQueryAction) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) TimeValue(org.opensearch.common.unit.TimeValue) RestHandlerUtils.createXContentParserFromRegistry(org.opensearch.ad.util.RestHandlerUtils.createXContentParserFromRegistry) SearchHit(org.opensearch.search.SearchHit) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) IS_LATEST_FIELD(org.opensearch.ad.model.ADTask.IS_LATEST_FIELD) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) ScoreMode(org.apache.lucene.search.join.ScoreMode) TransportService(org.opensearch.transport.TransportService) Logger(org.apache.logging.log4j.Logger) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) ADEntityTaskProfile(org.opensearch.ad.model.ADEntityTaskProfile) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HC_DETECTOR_TASK_IS_UPDATING(org.opensearch.ad.constant.CommonErrorMessages.HC_DETECTOR_TASK_IS_UPDATING) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ADCancelTaskAction(org.opensearch.ad.transport.ADCancelTaskAction) BiConsumer(java.util.function.BiConsumer) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) QueryBuilders(org.opensearch.index.query.QueryBuilders) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) IOException(java.io.IOException) TotalHits(org.apache.lucene.search.TotalHits) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ChronoUnit(java.time.temporal.ChronoUnit) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) ExceptionUtil.getErrorMessage(org.opensearch.ad.util.ExceptionUtil.getErrorMessage) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) COORDINATING_NODE_FIELD(org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) MAX_OLD_AD_TASK_DOCS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS) ADBatchAnomalyResultAction(org.opensearch.ad.transport.ADBatchAnomalyResultAction) ToXContent(org.opensearch.common.xcontent.ToXContent) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) BulkRequest(org.opensearch.action.bulk.BulkRequest) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) EXECUTION_START_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) DETECTOR_ID_FIELD(org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) REALTIME_TASK_TYPES(org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) DetectorProfile(org.opensearch.ad.model.DetectorProfile) ImmutableSet(com.google.common.collect.ImmutableSet) ESTIMATED_MINUTES_LEFT_FIELD(org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD) NOT_ENDED_STATES(org.opensearch.ad.model.ADTaskState.NOT_ENDED_STATES) ImmutableMap(com.google.common.collect.ImmutableMap) Script(org.opensearch.script.Script) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ADTaskType(org.opensearch.ad.model.ADTaskType) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) LAST_UPDATE_TIME_FIELD(org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TASK_TYPE_FIELD(org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD) Objects(java.util.Objects) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ALL_HISTORICAL_TASK_TYPES(org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) TASK_ID_FIELD(org.opensearch.ad.model.AnomalyResult.TASK_ID_FIELD) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) HashMap(java.util.HashMap) SortOrder(org.opensearch.search.sort.SortOrder) ImmutableList(com.google.common.collect.ImmutableList) EXCEED_HISTORICAL_ANALYSIS_LIMIT(org.opensearch.ad.constant.CommonErrorMessages.EXCEED_HISTORICAL_ANALYSIS_LIMIT) ADCancelTaskRequest(org.opensearch.ad.transport.ADCancelTaskRequest) SearchResponse(org.opensearch.action.search.SearchResponse) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) Iterator(java.util.Iterator) ALL_AD_RESULTS_INDEX_PATTERN(org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN) Semaphore(java.util.concurrent.Semaphore) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) AD_USED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_USED_BATCH_TASK_SLOT_COUNT) Entity(org.opensearch.ad.model.Entity) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ADTaskProfileAction(org.opensearch.ad.transport.ADTaskProfileAction) Script(org.opensearch.script.Script) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest)

Example 2 with PARENT_TASK_ID_FIELD

use of org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD in project anomaly-detection by opensearch-project.

the class ADTaskManager method getAndExecuteOnLatestADTasks.

/**
 * Get latest AD tasks and execute consumer function.
 * If resetTaskState is true, will collect latest task's profile data from all data nodes. If no data
 * node running the latest task, will reset the task state as STOPPED; otherwise, check if there is
 * any stale running entities(entity exists in coordinating node cache but no task running on worker
 * node) and clean up.
 * [Important!] Make sure listener returns in function
 *
 * @param detectorId detector id
 * @param parentTaskId parent task id
 * @param entity entity value
 * @param adTaskTypes AD task types
 * @param function consumer function
 * @param transportService transport service
 * @param resetTaskState reset task state or not
 * @param size return how many AD tasks
 * @param listener action listener
 * @param <T> response type of action listener
 */
public <T> void getAndExecuteOnLatestADTasks(String detectorId, String parentTaskId, Entity entity, List<ADTaskType> adTaskTypes, Consumer<List<ADTask>> function, TransportService transportService, boolean resetTaskState, int size, ActionListener<T> listener) {
    BoolQueryBuilder query = new BoolQueryBuilder();
    query.filter(new TermQueryBuilder(DETECTOR_ID_FIELD, detectorId));
    query.filter(new TermQueryBuilder(IS_LATEST_FIELD, true));
    if (parentTaskId != null) {
        query.filter(new TermQueryBuilder(PARENT_TASK_ID_FIELD, parentTaskId));
    }
    if (adTaskTypes != null && adTaskTypes.size() > 0) {
        query.filter(new TermsQueryBuilder(TASK_TYPE_FIELD, taskTypeToString(adTaskTypes)));
    }
    if (entity != null && !isNullOrEmpty(entity.getAttributes())) {
        String path = "entity";
        String entityKeyFieldName = path + ".name";
        String entityValueFieldName = path + ".value";
        for (Map.Entry<String, String> attribute : entity.getAttributes().entrySet()) {
            BoolQueryBuilder entityBoolQuery = new BoolQueryBuilder();
            TermQueryBuilder entityKeyFilterQuery = QueryBuilders.termQuery(entityKeyFieldName, attribute.getKey());
            TermQueryBuilder entityValueFilterQuery = QueryBuilders.termQuery(entityValueFieldName, attribute.getValue());
            entityBoolQuery.filter(entityKeyFilterQuery).filter(entityValueFilterQuery);
            NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(path, entityBoolQuery, ScoreMode.None);
            query.filter(nestedQueryBuilder);
        }
    }
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    sourceBuilder.query(query).sort(EXECUTION_START_TIME_FIELD, SortOrder.DESC).size(size);
    SearchRequest searchRequest = new SearchRequest();
    searchRequest.source(sourceBuilder);
    searchRequest.indices(DETECTION_STATE_INDEX);
    client.search(searchRequest, ActionListener.wrap(r -> {
        // https://github.com/opendistro-for-elasticsearch/anomaly-detection/pull/359#discussion_r558653132
        // getTotalHits will be null when we track_total_hits is false in the query request.
        // Add more checking here to cover some unknown cases.
        List<ADTask> adTasks = new ArrayList<>();
        if (r == null || r.getHits().getTotalHits() == null || r.getHits().getTotalHits().value == 0) {
            // don't throw exception here as consumer functions need to handle missing task
            // in different way.
            function.accept(adTasks);
            return;
        }
        Iterator<SearchHit> iterator = r.getHits().iterator();
        while (iterator.hasNext()) {
            SearchHit searchHit = iterator.next();
            try (XContentParser parser = createXContentParserFromRegistry(xContentRegistry, searchHit.getSourceRef())) {
                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
                ADTask adTask = ADTask.parse(parser, searchHit.getId());
                adTasks.add(adTask);
            } catch (Exception e) {
                String message = "Failed to parse AD task for detector " + detectorId + ", task id " + searchHit.getId();
                logger.error(message, e);
                listener.onFailure(new OpenSearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR));
            }
        }
        if (resetTaskState) {
            resetLatestDetectorTaskState(adTasks, function, transportService, listener);
        } else {
            function.accept(adTasks);
        }
    }, e -> {
        if (e instanceof IndexNotFoundException) {
            function.accept(new ArrayList<>());
        } else {
            logger.error("Failed to search AD task for detector " + detectorId, e);
            listener.onFailure(e);
        }
    }));
}
Also used : PARENT_TASK_ID_FIELD(org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) BulkAction(org.opensearch.action.bulk.BulkAction) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ExceptionUtil.getShardsFailure(org.opensearch.ad.util.ExceptionUtil.getShardsFailure) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) STOPPED_BY_FIELD(org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD) WriteRequest(org.opensearch.action.support.WriteRequest) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) ForwardADTaskAction(org.opensearch.ad.transport.ForwardADTaskAction) ActionListener(org.opensearch.action.ActionListener) DETECTOR_IS_RUNNING(org.opensearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) UpdateByQueryAction(org.opensearch.index.reindex.UpdateByQueryAction) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) TimeValue(org.opensearch.common.unit.TimeValue) RestHandlerUtils.createXContentParserFromRegistry(org.opensearch.ad.util.RestHandlerUtils.createXContentParserFromRegistry) SearchHit(org.opensearch.search.SearchHit) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) IS_LATEST_FIELD(org.opensearch.ad.model.ADTask.IS_LATEST_FIELD) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) ScoreMode(org.apache.lucene.search.join.ScoreMode) TransportService(org.opensearch.transport.TransportService) Logger(org.apache.logging.log4j.Logger) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) ADEntityTaskProfile(org.opensearch.ad.model.ADEntityTaskProfile) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HC_DETECTOR_TASK_IS_UPDATING(org.opensearch.ad.constant.CommonErrorMessages.HC_DETECTOR_TASK_IS_UPDATING) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ADCancelTaskAction(org.opensearch.ad.transport.ADCancelTaskAction) BiConsumer(java.util.function.BiConsumer) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) QueryBuilders(org.opensearch.index.query.QueryBuilders) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) IOException(java.io.IOException) TotalHits(org.apache.lucene.search.TotalHits) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ChronoUnit(java.time.temporal.ChronoUnit) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) ExceptionUtil.getErrorMessage(org.opensearch.ad.util.ExceptionUtil.getErrorMessage) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) COORDINATING_NODE_FIELD(org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) MAX_OLD_AD_TASK_DOCS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS) ADBatchAnomalyResultAction(org.opensearch.ad.transport.ADBatchAnomalyResultAction) ToXContent(org.opensearch.common.xcontent.ToXContent) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) BulkRequest(org.opensearch.action.bulk.BulkRequest) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) EXECUTION_START_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) DETECTOR_ID_FIELD(org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) REALTIME_TASK_TYPES(org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) DetectorProfile(org.opensearch.ad.model.DetectorProfile) ImmutableSet(com.google.common.collect.ImmutableSet) ESTIMATED_MINUTES_LEFT_FIELD(org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD) NOT_ENDED_STATES(org.opensearch.ad.model.ADTaskState.NOT_ENDED_STATES) ImmutableMap(com.google.common.collect.ImmutableMap) Script(org.opensearch.script.Script) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ADTaskType(org.opensearch.ad.model.ADTaskType) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) LAST_UPDATE_TIME_FIELD(org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TASK_TYPE_FIELD(org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD) Objects(java.util.Objects) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ALL_HISTORICAL_TASK_TYPES(org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) TASK_ID_FIELD(org.opensearch.ad.model.AnomalyResult.TASK_ID_FIELD) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) HashMap(java.util.HashMap) SortOrder(org.opensearch.search.sort.SortOrder) ImmutableList(com.google.common.collect.ImmutableList) EXCEED_HISTORICAL_ANALYSIS_LIMIT(org.opensearch.ad.constant.CommonErrorMessages.EXCEED_HISTORICAL_ANALYSIS_LIMIT) ADCancelTaskRequest(org.opensearch.ad.transport.ADCancelTaskRequest) SearchResponse(org.opensearch.action.search.SearchResponse) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) Iterator(java.util.Iterator) ALL_AD_RESULTS_INDEX_PATTERN(org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN) Semaphore(java.util.concurrent.Semaphore) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) AD_USED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_USED_BATCH_TASK_SLOT_COUNT) Entity(org.opensearch.ad.model.Entity) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ADTaskProfileAction(org.opensearch.ad.transport.ADTaskProfileAction) SearchRequest(org.opensearch.action.search.SearchRequest) SearchHit(org.opensearch.search.SearchHit) ArrayList(java.util.ArrayList) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) IOException(java.io.IOException) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) EndRunException(org.opensearch.ad.common.exception.EndRunException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) Iterator(java.util.Iterator) ADTask(org.opensearch.ad.model.ADTask) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) XContentParser(org.opensearch.common.xcontent.XContentParser) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Example 3 with PARENT_TASK_ID_FIELD

use of org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD in project anomaly-detection by opensearch-project.

the class ADTaskManager method countEntityTasksByState.

/**
 * Count entity tasks by state with detector level task id(parent task id).
 *
 * @param detectorTaskId detector level task id
 * @param taskStates task states
 * @param listener action listener
 */
public void countEntityTasksByState(String detectorTaskId, List<ADTaskState> taskStates, ActionListener<Long> listener) {
    BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
    queryBuilder.filter(new TermQueryBuilder(PARENT_TASK_ID_FIELD, detectorTaskId));
    if (taskStates != null && taskStates.size() > 0) {
        queryBuilder.filter(new TermsQueryBuilder(STATE_FIELD, taskStates.stream().map(s -> s.name()).collect(Collectors.toList())));
    }
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    sourceBuilder.query(queryBuilder);
    sourceBuilder.size(0);
    sourceBuilder.trackTotalHits(true);
    SearchRequest request = new SearchRequest();
    request.source(sourceBuilder);
    request.indices(DETECTION_STATE_INDEX);
    client.search(request, ActionListener.wrap(r -> {
        TotalHits totalHits = r.getHits().getTotalHits();
        listener.onResponse(totalHits.value);
    }, e -> listener.onFailure(e)));
}
Also used : PARENT_TASK_ID_FIELD(org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) BulkAction(org.opensearch.action.bulk.BulkAction) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ExceptionUtil.getShardsFailure(org.opensearch.ad.util.ExceptionUtil.getShardsFailure) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) STOPPED_BY_FIELD(org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD) WriteRequest(org.opensearch.action.support.WriteRequest) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) ForwardADTaskAction(org.opensearch.ad.transport.ForwardADTaskAction) ActionListener(org.opensearch.action.ActionListener) DETECTOR_IS_RUNNING(org.opensearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) UpdateByQueryAction(org.opensearch.index.reindex.UpdateByQueryAction) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) TimeValue(org.opensearch.common.unit.TimeValue) RestHandlerUtils.createXContentParserFromRegistry(org.opensearch.ad.util.RestHandlerUtils.createXContentParserFromRegistry) SearchHit(org.opensearch.search.SearchHit) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) IS_LATEST_FIELD(org.opensearch.ad.model.ADTask.IS_LATEST_FIELD) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) ScoreMode(org.apache.lucene.search.join.ScoreMode) TransportService(org.opensearch.transport.TransportService) Logger(org.apache.logging.log4j.Logger) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) ADEntityTaskProfile(org.opensearch.ad.model.ADEntityTaskProfile) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HC_DETECTOR_TASK_IS_UPDATING(org.opensearch.ad.constant.CommonErrorMessages.HC_DETECTOR_TASK_IS_UPDATING) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ADCancelTaskAction(org.opensearch.ad.transport.ADCancelTaskAction) BiConsumer(java.util.function.BiConsumer) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) QueryBuilders(org.opensearch.index.query.QueryBuilders) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) IOException(java.io.IOException) TotalHits(org.apache.lucene.search.TotalHits) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ChronoUnit(java.time.temporal.ChronoUnit) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) ExceptionUtil.getErrorMessage(org.opensearch.ad.util.ExceptionUtil.getErrorMessage) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) COORDINATING_NODE_FIELD(org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) MAX_OLD_AD_TASK_DOCS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS) ADBatchAnomalyResultAction(org.opensearch.ad.transport.ADBatchAnomalyResultAction) ToXContent(org.opensearch.common.xcontent.ToXContent) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) BulkRequest(org.opensearch.action.bulk.BulkRequest) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) EXECUTION_START_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) DETECTOR_ID_FIELD(org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) REALTIME_TASK_TYPES(org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) DetectorProfile(org.opensearch.ad.model.DetectorProfile) ImmutableSet(com.google.common.collect.ImmutableSet) ESTIMATED_MINUTES_LEFT_FIELD(org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD) NOT_ENDED_STATES(org.opensearch.ad.model.ADTaskState.NOT_ENDED_STATES) ImmutableMap(com.google.common.collect.ImmutableMap) Script(org.opensearch.script.Script) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ADTaskType(org.opensearch.ad.model.ADTaskType) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) LAST_UPDATE_TIME_FIELD(org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TASK_TYPE_FIELD(org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD) Objects(java.util.Objects) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ALL_HISTORICAL_TASK_TYPES(org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) TASK_ID_FIELD(org.opensearch.ad.model.AnomalyResult.TASK_ID_FIELD) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) HashMap(java.util.HashMap) SortOrder(org.opensearch.search.sort.SortOrder) ImmutableList(com.google.common.collect.ImmutableList) EXCEED_HISTORICAL_ANALYSIS_LIMIT(org.opensearch.ad.constant.CommonErrorMessages.EXCEED_HISTORICAL_ANALYSIS_LIMIT) ADCancelTaskRequest(org.opensearch.ad.transport.ADCancelTaskRequest) SearchResponse(org.opensearch.action.search.SearchResponse) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) Iterator(java.util.Iterator) ALL_AD_RESULTS_INDEX_PATTERN(org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN) Semaphore(java.util.concurrent.Semaphore) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) AD_USED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_USED_BATCH_TASK_SLOT_COUNT) Entity(org.opensearch.ad.model.Entity) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ADTaskProfileAction(org.opensearch.ad.transport.ADTaskProfileAction) TotalHits(org.apache.lucene.search.TotalHits) SearchRequest(org.opensearch.action.search.SearchRequest) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder)

Example 4 with PARENT_TASK_ID_FIELD

use of org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD in project anomaly-detection by opensearch-project.

the class ADTaskManager method cleanChildTasksAndADResultsOfDeletedTask.

/**
 * Poll deleted detector task from cache and delete its child tasks and AD results.
 */
public void cleanChildTasksAndADResultsOfDeletedTask() {
    if (!adTaskCacheManager.hasDeletedDetectorTask()) {
        return;
    }
    threadPool.schedule(() -> {
        String taskId = adTaskCacheManager.pollDeletedDetectorTask();
        if (taskId == null) {
            return;
        }
        DeleteByQueryRequest deleteADResultsRequest = new DeleteByQueryRequest(ALL_AD_RESULTS_INDEX_PATTERN);
        deleteADResultsRequest.setQuery(new TermsQueryBuilder(TASK_ID_FIELD, taskId));
        client.execute(DeleteByQueryAction.INSTANCE, deleteADResultsRequest, ActionListener.wrap(res -> {
            logger.debug("Successfully deleted AD results of task " + taskId);
            DeleteByQueryRequest deleteChildTasksRequest = new DeleteByQueryRequest(DETECTION_STATE_INDEX);
            deleteChildTasksRequest.setQuery(new TermsQueryBuilder(PARENT_TASK_ID_FIELD, taskId));
            client.execute(DeleteByQueryAction.INSTANCE, deleteChildTasksRequest, ActionListener.wrap(r -> {
                logger.debug("Successfully deleted child tasks of task " + taskId);
                cleanChildTasksAndADResultsOfDeletedTask();
            }, e -> {
                logger.error("Failed to delete child tasks of task " + taskId, e);
            }));
        }, ex -> {
            logger.error("Failed to delete AD results for task " + taskId, ex);
        }));
    }, TimeValue.timeValueSeconds(DEFAULT_MAINTAIN_INTERVAL_IN_SECONDS), AD_BATCH_TASK_THREAD_POOL_NAME);
}
Also used : PARENT_TASK_ID_FIELD(org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) BulkAction(org.opensearch.action.bulk.BulkAction) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ExceptionUtil.getShardsFailure(org.opensearch.ad.util.ExceptionUtil.getShardsFailure) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) STOPPED_BY_FIELD(org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD) WriteRequest(org.opensearch.action.support.WriteRequest) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) ForwardADTaskAction(org.opensearch.ad.transport.ForwardADTaskAction) ActionListener(org.opensearch.action.ActionListener) DETECTOR_IS_RUNNING(org.opensearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) UpdateByQueryAction(org.opensearch.index.reindex.UpdateByQueryAction) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) TimeValue(org.opensearch.common.unit.TimeValue) RestHandlerUtils.createXContentParserFromRegistry(org.opensearch.ad.util.RestHandlerUtils.createXContentParserFromRegistry) SearchHit(org.opensearch.search.SearchHit) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) IS_LATEST_FIELD(org.opensearch.ad.model.ADTask.IS_LATEST_FIELD) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) ScoreMode(org.apache.lucene.search.join.ScoreMode) TransportService(org.opensearch.transport.TransportService) Logger(org.apache.logging.log4j.Logger) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) ADEntityTaskProfile(org.opensearch.ad.model.ADEntityTaskProfile) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HC_DETECTOR_TASK_IS_UPDATING(org.opensearch.ad.constant.CommonErrorMessages.HC_DETECTOR_TASK_IS_UPDATING) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ADCancelTaskAction(org.opensearch.ad.transport.ADCancelTaskAction) BiConsumer(java.util.function.BiConsumer) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) QueryBuilders(org.opensearch.index.query.QueryBuilders) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) IOException(java.io.IOException) TotalHits(org.apache.lucene.search.TotalHits) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ChronoUnit(java.time.temporal.ChronoUnit) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) ExceptionUtil.getErrorMessage(org.opensearch.ad.util.ExceptionUtil.getErrorMessage) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) COORDINATING_NODE_FIELD(org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) MAX_OLD_AD_TASK_DOCS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS) ADBatchAnomalyResultAction(org.opensearch.ad.transport.ADBatchAnomalyResultAction) ToXContent(org.opensearch.common.xcontent.ToXContent) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) BulkRequest(org.opensearch.action.bulk.BulkRequest) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) EXECUTION_START_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) DETECTOR_ID_FIELD(org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) REALTIME_TASK_TYPES(org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) DetectorProfile(org.opensearch.ad.model.DetectorProfile) ImmutableSet(com.google.common.collect.ImmutableSet) ESTIMATED_MINUTES_LEFT_FIELD(org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD) NOT_ENDED_STATES(org.opensearch.ad.model.ADTaskState.NOT_ENDED_STATES) ImmutableMap(com.google.common.collect.ImmutableMap) Script(org.opensearch.script.Script) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ADTaskType(org.opensearch.ad.model.ADTaskType) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) LAST_UPDATE_TIME_FIELD(org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TASK_TYPE_FIELD(org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD) Objects(java.util.Objects) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ALL_HISTORICAL_TASK_TYPES(org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) TASK_ID_FIELD(org.opensearch.ad.model.AnomalyResult.TASK_ID_FIELD) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) HashMap(java.util.HashMap) SortOrder(org.opensearch.search.sort.SortOrder) ImmutableList(com.google.common.collect.ImmutableList) EXCEED_HISTORICAL_ANALYSIS_LIMIT(org.opensearch.ad.constant.CommonErrorMessages.EXCEED_HISTORICAL_ANALYSIS_LIMIT) ADCancelTaskRequest(org.opensearch.ad.transport.ADCancelTaskRequest) SearchResponse(org.opensearch.action.search.SearchResponse) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) Iterator(java.util.Iterator) ALL_AD_RESULTS_INDEX_PATTERN(org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN) Semaphore(java.util.concurrent.Semaphore) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) AD_USED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_USED_BATCH_TASK_SLOT_COUNT) Entity(org.opensearch.ad.model.Entity) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ADTaskProfileAction(org.opensearch.ad.transport.ADTaskProfileAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)4 ImmutableMap (com.google.common.collect.ImmutableMap)4 ImmutableSet (com.google.common.collect.ImmutableSet)4 IOException (java.io.IOException)4 Instant (java.time.Instant)4 ChronoUnit (java.time.temporal.ChronoUnit)4 ArrayList (java.util.ArrayList)4 HashMap (java.util.HashMap)4 Iterator (java.util.Iterator)4 List (java.util.List)4 Locale (java.util.Locale)4 Map (java.util.Map)4 Objects (java.util.Objects)4 Optional (java.util.Optional)4 Set (java.util.Set)4 ConcurrentLinkedQueue (java.util.concurrent.ConcurrentLinkedQueue)4 Semaphore (java.util.concurrent.Semaphore)4 BiConsumer (java.util.function.BiConsumer)4 Consumer (java.util.function.Consumer)4 Collectors (java.util.stream.Collectors)4