Search in sources :

Example 16 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class RestAnomalyDetectorJobAction method parseDetectionDateRange.

private DetectionDateRange parseDetectionDateRange(RestRequest request) throws IOException {
    if (!request.hasContent()) {
        return null;
    }
    XContentParser parser = request.contentParser();
    ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
    DetectionDateRange dateRange = DetectionDateRange.parse(parser);
    return dateRange;
}
Also used : XContentParser(org.opensearch.common.xcontent.XContentParser) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange)

Example 17 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class ADTaskManager method createNewADTask.

private void createNewADTask(AnomalyDetector detector, DetectionDateRange detectionDateRange, User user, String coordinatingNode, ActionListener<AnomalyDetectorJobResponse> listener) {
    String userName = user == null ? null : user.getName();
    Instant now = Instant.now();
    String taskType = getADTaskType(detector, detectionDateRange).name();
    ADTask adTask = new ADTask.Builder().detectorId(detector.getDetectorId()).detector(detector).isLatest(true).taskType(taskType).executionStartTime(now).taskProgress(0.0f).initProgress(0.0f).state(ADTaskState.CREATED.name()).lastUpdateTime(now).startedBy(userName).coordinatingNode(coordinatingNode).detectionDateRange(detectionDateRange).user(user).build();
    createADTaskDirectly(adTask, r -> onIndexADTaskResponse(r, adTask, (response, delegatedListener) -> cleanOldAdTaskDocs(response, adTask, delegatedListener), listener), listener);
}
Also used : PARENT_TASK_ID_FIELD(org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) BulkAction(org.opensearch.action.bulk.BulkAction) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ExceptionUtil.getShardsFailure(org.opensearch.ad.util.ExceptionUtil.getShardsFailure) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) STOPPED_BY_FIELD(org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD) WriteRequest(org.opensearch.action.support.WriteRequest) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) ForwardADTaskAction(org.opensearch.ad.transport.ForwardADTaskAction) ActionListener(org.opensearch.action.ActionListener) DETECTOR_IS_RUNNING(org.opensearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) UpdateByQueryAction(org.opensearch.index.reindex.UpdateByQueryAction) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) TimeValue(org.opensearch.common.unit.TimeValue) RestHandlerUtils.createXContentParserFromRegistry(org.opensearch.ad.util.RestHandlerUtils.createXContentParserFromRegistry) SearchHit(org.opensearch.search.SearchHit) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) IS_LATEST_FIELD(org.opensearch.ad.model.ADTask.IS_LATEST_FIELD) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) ScoreMode(org.apache.lucene.search.join.ScoreMode) TransportService(org.opensearch.transport.TransportService) Logger(org.apache.logging.log4j.Logger) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) ADEntityTaskProfile(org.opensearch.ad.model.ADEntityTaskProfile) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HC_DETECTOR_TASK_IS_UPDATING(org.opensearch.ad.constant.CommonErrorMessages.HC_DETECTOR_TASK_IS_UPDATING) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ADCancelTaskAction(org.opensearch.ad.transport.ADCancelTaskAction) BiConsumer(java.util.function.BiConsumer) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) QueryBuilders(org.opensearch.index.query.QueryBuilders) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) IOException(java.io.IOException) TotalHits(org.apache.lucene.search.TotalHits) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ChronoUnit(java.time.temporal.ChronoUnit) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) ExceptionUtil.getErrorMessage(org.opensearch.ad.util.ExceptionUtil.getErrorMessage) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) COORDINATING_NODE_FIELD(org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) MAX_OLD_AD_TASK_DOCS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS) ADBatchAnomalyResultAction(org.opensearch.ad.transport.ADBatchAnomalyResultAction) ToXContent(org.opensearch.common.xcontent.ToXContent) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) BulkRequest(org.opensearch.action.bulk.BulkRequest) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) EXECUTION_START_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) DETECTOR_ID_FIELD(org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) REALTIME_TASK_TYPES(org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) DetectorProfile(org.opensearch.ad.model.DetectorProfile) ImmutableSet(com.google.common.collect.ImmutableSet) ESTIMATED_MINUTES_LEFT_FIELD(org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD) NOT_ENDED_STATES(org.opensearch.ad.model.ADTaskState.NOT_ENDED_STATES) ImmutableMap(com.google.common.collect.ImmutableMap) Script(org.opensearch.script.Script) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ADTaskType(org.opensearch.ad.model.ADTaskType) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) LAST_UPDATE_TIME_FIELD(org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TASK_TYPE_FIELD(org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD) Objects(java.util.Objects) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ALL_HISTORICAL_TASK_TYPES(org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) TASK_ID_FIELD(org.opensearch.ad.model.AnomalyResult.TASK_ID_FIELD) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) HashMap(java.util.HashMap) SortOrder(org.opensearch.search.sort.SortOrder) ImmutableList(com.google.common.collect.ImmutableList) EXCEED_HISTORICAL_ANALYSIS_LIMIT(org.opensearch.ad.constant.CommonErrorMessages.EXCEED_HISTORICAL_ANALYSIS_LIMIT) ADCancelTaskRequest(org.opensearch.ad.transport.ADCancelTaskRequest) SearchResponse(org.opensearch.action.search.SearchResponse) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) Iterator(java.util.Iterator) ALL_AD_RESULTS_INDEX_PATTERN(org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN) Semaphore(java.util.concurrent.Semaphore) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) AD_USED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_USED_BATCH_TASK_SLOT_COUNT) Entity(org.opensearch.ad.model.Entity) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ADTaskProfileAction(org.opensearch.ad.transport.ADTaskProfileAction) Instant(java.time.Instant) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) ADTask(org.opensearch.ad.model.ADTask) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString)

Example 18 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class ForwardADTaskTransportAction method doExecute.

@Override
protected void doExecute(Task task, ForwardADTaskRequest request, ActionListener<AnomalyDetectorJobResponse> listener) {
    ADTaskAction adTaskAction = request.getAdTaskAction();
    AnomalyDetector detector = request.getDetector();
    DetectionDateRange detectionDateRange = request.getDetectionDateRange();
    String detectorId = detector.getDetectorId();
    ADTask adTask = request.getAdTask();
    User user = request.getUser();
    Integer availableTaskSlots = request.getAvailableTaskSLots();
    String entityValue = adTaskManager.convertEntityToString(adTask);
    switch(adTaskAction) {
        case APPLY_FOR_TASK_SLOTS:
            logger.debug("Received APPLY_FOR_TASK_SLOTS action for detector {}", detectorId);
            adTaskManager.checkTaskSlots(adTask, detector, detectionDateRange, user, ADTaskAction.START, transportService, listener);
            break;
        case CHECK_AVAILABLE_TASK_SLOTS:
            logger.debug("Received CHECK_AVAILABLE_TASK_SLOTS action for detector {}", detectorId);
            adTaskManager.checkTaskSlots(adTask, detector, detectionDateRange, user, ADTaskAction.SCALE_ENTITY_TASK_SLOTS, transportService, listener);
            break;
        case START:
            // Start historical analysis for detector
            logger.debug("Received START action for detector {}", detectorId);
            adTaskManager.startDetector(detector, detectionDateRange, user, transportService, ActionListener.wrap(r -> {
                adTaskCacheManager.setDetectorTaskSlots(detector.getDetectorId(), availableTaskSlots);
                listener.onResponse(r);
            }, e -> listener.onFailure(e)));
            break;
        case NEXT_ENTITY:
            logger.debug("Received NEXT_ENTITY action for detector {}, task {}", detectorId, adTask.getTaskId());
            // Run next entity for HC detector historical analysis.
            if (detector.isMultientityDetector()) {
                // AD task could be HC detector level task or entity task
                adTaskCacheManager.removeRunningEntity(detectorId, entityValue);
                if (!adTaskCacheManager.hasEntity(detectorId)) {
                    adTaskCacheManager.setDetectorTaskSlots(detectorId, 0);
                    logger.info("Historical HC detector done, will remove from cache, detector id:{}", detectorId);
                    listener.onResponse(new AnomalyDetectorJobResponse(detectorId, 0, 0, 0, RestStatus.OK));
                    ADTaskState state = !adTask.isEntityTask() && adTask.getError() != null ? ADTaskState.FAILED : ADTaskState.FINISHED;
                    adTaskManager.setHCDetectorTaskDone(adTask, state, listener);
                } else {
                    logger.debug("Run next entity for detector " + detectorId);
                    adTaskManager.runNextEntityForHCADHistorical(adTask, transportService, listener);
                    adTaskManager.updateADHCDetectorTask(detectorId, adTask.getParentTaskId(), ImmutableMap.of(STATE_FIELD, ADTaskState.RUNNING.name(), TASK_PROGRESS_FIELD, adTaskManager.hcDetectorProgress(detectorId), ERROR_FIELD, adTask.getError() != null ? adTask.getError() : ""));
                }
            } else {
                logger.warn("Can only handle HC entity task for NEXT_ENTITY action, taskId:{} , taskType:{}", adTask.getTaskId(), adTask.getTaskType());
                listener.onFailure(new IllegalArgumentException("Unsupported task"));
            }
            break;
        case PUSH_BACK_ENTITY:
            logger.debug("Received PUSH_BACK_ENTITY action for detector {}, task {}", detectorId, adTask.getTaskId());
            // Push back entity to pending entities queue and run next entity.
            if (adTask.isEntityTask()) {
                // AD task must be entity level task.
                adTaskCacheManager.removeRunningEntity(detectorId, entityValue);
                if (adTaskManager.isRetryableError(adTask.getError()) && !adTaskCacheManager.exceedRetryLimit(adTask.getDetectorId(), adTask.getTaskId())) {
                    // If retryable exception happens when run entity task, will push back entity to the end
                    // of pending entities queue, then we can retry it later.
                    adTaskCacheManager.pushBackEntity(adTask.getTaskId(), adTask.getDetectorId(), entityValue);
                } else {
                    // If exception is not retryable or exceeds retry limit, will remove this entity.
                    adTaskCacheManager.removeEntity(adTask.getDetectorId(), entityValue);
                    logger.warn("Entity task failed, task id: {}, entity: {}", adTask.getTaskId(), adTask.getEntity().toString());
                }
                if (!adTaskCacheManager.hasEntity(detectorId)) {
                    adTaskCacheManager.setDetectorTaskSlots(detectorId, 0);
                    adTaskManager.setHCDetectorTaskDone(adTask, ADTaskState.FINISHED, listener);
                } else {
                    logger.debug("scale task slots for PUSH_BACK_ENTITY, detector {} task {}", detectorId, adTask.getTaskId());
                    int taskSlots = adTaskCacheManager.scaleDownHCDetectorTaskSlots(detectorId, 1);
                    if (taskSlots == 1) {
                        logger.debug("After scale down, only 1 task slot reserved for detector {}, run next entity", detectorId);
                        adTaskManager.runNextEntityForHCADHistorical(adTask, transportService, listener);
                    }
                    listener.onResponse(new AnomalyDetectorJobResponse(adTask.getTaskId(), 0, 0, 0, RestStatus.ACCEPTED));
                }
            } else {
                logger.warn("Can only push back entity task");
                listener.onFailure(new IllegalArgumentException("Can only push back entity task"));
            }
            break;
        case SCALE_ENTITY_TASK_SLOTS:
            logger.debug("Received SCALE_ENTITY_TASK_LANE action for detector {}", detectorId);
            // Check current available task slots and scale entity task lane.
            if (availableTaskSlots != null && availableTaskSlots > 0) {
                int newSlots = Math.min(availableTaskSlots, adTaskManager.detectorTaskSlotScaleDelta(detectorId));
                if (newSlots > 0) {
                    adTaskCacheManager.setAllowedRunningEntities(detectorId, newSlots);
                    adTaskCacheManager.scaleUpDetectorTaskSlots(detectorId, newSlots);
                }
            }
            listener.onResponse(new AnomalyDetectorJobResponse(detector.getDetectorId(), 0, 0, 0, RestStatus.OK));
            break;
        case CANCEL:
            logger.debug("Received CANCEL action for detector {}", detectorId);
            // on worker node.
            if (detector.isMultientityDetector()) {
                adTaskCacheManager.clearPendingEntities(detectorId);
                adTaskCacheManager.removeRunningEntity(detectorId, entityValue);
                if (!adTaskCacheManager.hasEntity(detectorId) || !adTask.isEntityTask()) {
                    adTaskManager.setHCDetectorTaskDone(adTask, ADTaskState.STOPPED, listener);
                }
                listener.onResponse(new AnomalyDetectorJobResponse(adTask.getTaskId(), 0, 0, 0, RestStatus.OK));
            } else {
                listener.onFailure(new IllegalArgumentException("Only support cancel HC now"));
            }
            break;
        case CLEAN_STALE_RUNNING_ENTITIES:
            logger.debug("Received CLEAN_STALE_RUNNING_ENTITIES action for detector {}", detectorId);
            // Clean stale running entities of HC detector. For example, some worker node crashed or failed to send
            // entity task done message to coordinating node, then coordinating node can't remove running entity
            // from cache. We will check task profile when get task. If some entities exist in coordinating cache but
            // doesn't exist in worker node's cache, we will clean up these stale running entities on coordinating node.
            List<String> staleRunningEntities = request.getStaleRunningEntities();
            logger.debug("Clean stale running entities of task {}, staleRunningEntities: {}", adTask.getTaskId(), Arrays.toString(staleRunningEntities.toArray(new String[0])));
            for (String entity : staleRunningEntities) {
                adTaskManager.removeStaleRunningEntity(adTask, entity, transportService, listener);
            }
            listener.onResponse(new AnomalyDetectorJobResponse(adTask.getTaskId(), 0, 0, 0, RestStatus.OK));
            break;
        case CLEAN_CACHE:
            boolean historicalTask = adTask.isHistoricalTask();
            logger.debug("Received CLEAN_CACHE action for detector {}, taskId: {}, historical: {}", detectorId, adTask.getTaskId(), historicalTask);
            if (historicalTask) {
                // Don't clear task cache if still has running entity. CLEAN_STALE_RUNNING_ENTITIES will clean
                // stale running entity.
                adTaskCacheManager.removeHistoricalTaskCacheIfNoRunningEntity(detectorId);
            } else {
                adTaskCacheManager.removeRealtimeTaskCache(detectorId);
                // If hash ring changed like new node added when scale out, the realtime job coordinating node may
                // change, then we should clean up cache on old coordinating node.
                stateManager.clear(detectorId);
                featureManager.clear(detectorId);
            }
            listener.onResponse(new AnomalyDetectorJobResponse(detector.getDetectorId(), 0, 0, 0, RestStatus.OK));
            break;
        default:
            listener.onFailure(new OpenSearchStatusException("Unsupported AD task action " + adTaskAction, RestStatus.BAD_REQUEST));
            break;
    }
}
Also used : ADTaskCacheManager(org.opensearch.ad.task.ADTaskCacheManager) Arrays(java.util.Arrays) HandledTransportAction(org.opensearch.action.support.HandledTransportAction) ADTask(org.opensearch.ad.model.ADTask) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) FeatureManager(org.opensearch.ad.feature.FeatureManager) ImmutableMap(com.google.common.collect.ImmutableMap) ADTaskManager(org.opensearch.ad.task.ADTaskManager) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) Task(org.opensearch.tasks.Task) RestStatus(org.opensearch.rest.RestStatus) TransportService(org.opensearch.transport.TransportService) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) NodeStateManager(org.opensearch.ad.NodeStateManager) User(org.opensearch.commons.authuser.User) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) ADTaskAction(org.opensearch.ad.model.ADTaskAction) LogManager(org.apache.logging.log4j.LogManager) User(org.opensearch.commons.authuser.User) ADTaskAction(org.opensearch.ad.model.ADTaskAction) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) ADTask(org.opensearch.ad.model.ADTask) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Example 19 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobTransportAction method doExecute.

@Override
protected void doExecute(Task task, AnomalyDetectorJobRequest request, ActionListener<AnomalyDetectorJobResponse> actionListener) {
    String detectorId = request.getDetectorID();
    DetectionDateRange detectionDateRange = request.getDetectionDateRange();
    boolean historical = request.isHistorical();
    long seqNo = request.getSeqNo();
    long primaryTerm = request.getPrimaryTerm();
    String rawPath = request.getRawPath();
    TimeValue requestTimeout = REQUEST_TIMEOUT.get(settings);
    String errorMessage = rawPath.endsWith(RestHandlerUtils.START_JOB) ? FAIL_TO_START_DETECTOR : FAIL_TO_STOP_DETECTOR;
    ActionListener<AnomalyDetectorJobResponse> listener = wrapRestActionListener(actionListener, errorMessage);
    // By the time request reaches here, the user permissions are validated by Security plugin.
    User user = getUserContext(client);
    try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
        resolveUserAndExecute(user, detectorId, filterByEnabled, listener, (anomalyDetector) -> executeDetector(listener, detectorId, detectionDateRange, historical, seqNo, primaryTerm, rawPath, requestTimeout, user, context), client, clusterService, xContentRegistry);
    } catch (Exception e) {
        logger.error(e);
        listener.onFailure(e);
    }
}
Also used : User(org.opensearch.commons.authuser.User) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) TimeValue(org.opensearch.common.unit.TimeValue)

Example 20 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class HistoricalAnalysisRestTestCase method startHistoricalAnalysis.

protected String startHistoricalAnalysis(String detectorId) throws IOException {
    Instant endTime = Instant.now().truncatedTo(ChronoUnit.SECONDS);
    Instant startTime = endTime.minus(10, ChronoUnit.DAYS).truncatedTo(ChronoUnit.SECONDS);
    DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime);
    Response startDetectorResponse = startAnomalyDetector(detectorId, dateRange, client());
    Map<String, Object> startDetectorResponseMap = responseAsMap(startDetectorResponse);
    String taskId = (String) startDetectorResponseMap.get("_id");
    assertNotNull(taskId);
    return taskId;
}
Also used : Response(org.opensearch.client.Response) Instant(java.time.Instant) ToXContentObject(org.opensearch.common.xcontent.ToXContentObject) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange)

Aggregations

DetectionDateRange (org.opensearch.ad.model.DetectionDateRange)32 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)14 Instant (java.time.Instant)13 IOException (java.io.IOException)8 AnomalyDetectorJobResponse (org.opensearch.ad.transport.AnomalyDetectorJobResponse)8 TimeValue (org.opensearch.common.unit.TimeValue)8 List (java.util.List)7 ActionListener (org.opensearch.action.ActionListener)7 EndRunException (org.opensearch.ad.common.exception.EndRunException)7 ClusterService (org.opensearch.cluster.service.ClusterService)7 ImmutableList (com.google.common.collect.ImmutableList)6 ImmutableMap (com.google.common.collect.ImmutableMap)6 Optional (java.util.Optional)6 LogManager (org.apache.logging.log4j.LogManager)6 Logger (org.apache.logging.log4j.Logger)6 HashRing (org.opensearch.ad.cluster.HashRing)6 AnomalyDetectionIndices (org.opensearch.ad.indices.AnomalyDetectionIndices)6 ADTask (org.opensearch.ad.model.ADTask)6 User (org.opensearch.commons.authuser.User)6 ImmutableSet (com.google.common.collect.ImmutableSet)5