Search in sources :

Example 6 with ADTaskProfile

use of org.opensearch.ad.model.ADTaskProfile in project anomaly-detection by opensearch-project.

the class ADTaskProfileTests method testADTaskProfileNodeResponse.

public void testADTaskProfileNodeResponse() throws IOException {
    ADTaskProfile adTaskProfile = new ADTaskProfile(randomAlphaOfLength(5), randomInt(), randomLong(), randomBoolean(), randomInt(), randomLong(), randomAlphaOfLength(5));
    ADTaskProfileNodeResponse response = new ADTaskProfileNodeResponse(randomDiscoveryNode(), adTaskProfile, Version.CURRENT);
    testADTaskProfileResponse(response);
}
Also used : ADTaskProfile(org.opensearch.ad.model.ADTaskProfile)

Example 7 with ADTaskProfile

use of org.opensearch.ad.model.ADTaskProfile in project anomaly-detection by opensearch-project.

the class HistoricalAnalysisRestApiIT method checkIfTaskCanFinishCorrectly.

private void checkIfTaskCanFinishCorrectly(String detectorId, String taskId, Set<String> states) throws InterruptedException {
    List<Object> results = waitUntilTaskDone(detectorId);
    ADTaskProfile endTaskProfile = (ADTaskProfile) results.get(0);
    Integer retryCount = (Integer) results.get(1);
    ADTask stoppedAdTask = endTaskProfile.getAdTask();
    assertEquals(taskId, stoppedAdTask.getTaskId());
    if (retryCount < MAX_RETRY_TIMES) {
        // It's possible that historical analysis still running after max retry times
        assertTrue(states.contains(stoppedAdTask.getState()));
    }
}
Also used : ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ADTask(org.opensearch.ad.model.ADTask) ToXContentObject(org.opensearch.common.xcontent.ToXContentObject)

Example 8 with ADTaskProfile

use of org.opensearch.ad.model.ADTaskProfile in project anomaly-detection by opensearch-project.

the class HistoricalAnalysisRestTestCase method parseADTaskProfile.

public ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException {
    String profileResult = EntityUtils.toString(profileResponse.getEntity());
    XContentParser parser = TestHelpers.parser(profileResult);
    ADTaskProfile adTaskProfile = null;
    while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
        String fieldName = parser.currentName();
        parser.nextToken();
        if ("ad_task".equals(fieldName)) {
            adTaskProfile = ADTaskProfile.parse(parser);
        } else {
            parser.skipChildren();
        }
    }
    return adTaskProfile;
}
Also used : ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) XContentParser(org.opensearch.common.xcontent.XContentParser)

Example 9 with ADTaskProfile

use of org.opensearch.ad.model.ADTaskProfile in project anomaly-detection by opensearch-project.

the class ADRestTestUtils method parseADTaskProfile.

public static ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException {
    String profileResult = EntityUtils.toString(profileResponse.getEntity());
    XContentParser parser = TestHelpers.parser(profileResult);
    ADTaskProfile adTaskProfile = null;
    while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
        String fieldName = parser.currentName();
        parser.nextToken();
        if ("ad_task".equals(fieldName)) {
            adTaskProfile = ADTaskProfile.parse(parser);
        } else {
            parser.skipChildren();
        }
    }
    return adTaskProfile;
}
Also used : ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) XContentParser(org.opensearch.common.xcontent.XContentParser)

Example 10 with ADTaskProfile

use of org.opensearch.ad.model.ADTaskProfile in project anomaly-detection by opensearch-project.

the class ADTaskManager method getADTaskProfile.

/**
 * Get AD task profile.
 * @param adDetectorLevelTask detector level task
 * @param listener action listener
 */
private void getADTaskProfile(ADTask adDetectorLevelTask, ActionListener<ADTaskProfile> listener) {
    String detectorId = adDetectorLevelTask.getDetectorId();
    hashRing.getAllEligibleDataNodesWithKnownAdVersion(dataNodes -> {
        ADTaskProfileRequest adTaskProfileRequest = new ADTaskProfileRequest(detectorId, dataNodes);
        client.execute(ADTaskProfileAction.INSTANCE, adTaskProfileRequest, ActionListener.wrap(response -> {
            if (response.hasFailures()) {
                listener.onFailure(response.failures().get(0));
                return;
            }
            List<ADEntityTaskProfile> adEntityTaskProfiles = new ArrayList<>();
            ADTaskProfile detectorTaskProfile = new ADTaskProfile(adDetectorLevelTask);
            for (ADTaskProfileNodeResponse node : response.getNodes()) {
                ADTaskProfile taskProfile = node.getAdTaskProfile();
                if (taskProfile != null) {
                    if (taskProfile.getNodeId() != null) {
                        // HC detector: task profile from coordinating node
                        // Single entity detector: task profile from worker node
                        detectorTaskProfile.setTaskId(taskProfile.getTaskId());
                        detectorTaskProfile.setShingleSize(taskProfile.getShingleSize());
                        detectorTaskProfile.setRcfTotalUpdates(taskProfile.getRcfTotalUpdates());
                        detectorTaskProfile.setThresholdModelTrained(taskProfile.getThresholdModelTrained());
                        detectorTaskProfile.setThresholdModelTrainingDataSize(taskProfile.getThresholdModelTrainingDataSize());
                        detectorTaskProfile.setModelSizeInBytes(taskProfile.getModelSizeInBytes());
                        detectorTaskProfile.setNodeId(taskProfile.getNodeId());
                        detectorTaskProfile.setTotalEntitiesCount(taskProfile.getTotalEntitiesCount());
                        detectorTaskProfile.setDetectorTaskSlots(taskProfile.getDetectorTaskSlots());
                        detectorTaskProfile.setPendingEntitiesCount(taskProfile.getPendingEntitiesCount());
                        detectorTaskProfile.setRunningEntitiesCount(taskProfile.getRunningEntitiesCount());
                        detectorTaskProfile.setRunningEntities(taskProfile.getRunningEntities());
                        detectorTaskProfile.setAdTaskType(taskProfile.getAdTaskType());
                    }
                    if (taskProfile.getEntityTaskProfiles() != null) {
                        adEntityTaskProfiles.addAll(taskProfile.getEntityTaskProfiles());
                    }
                }
            }
            if (adEntityTaskProfiles != null && adEntityTaskProfiles.size() > 0) {
                detectorTaskProfile.setEntityTaskProfiles(adEntityTaskProfiles);
            }
            listener.onResponse(detectorTaskProfile);
        }, e -> {
            logger.error("Failed to get task profile for task " + adDetectorLevelTask.getTaskId(), e);
            listener.onFailure(e);
        }));
    }, listener);
}
Also used : PARENT_TASK_ID_FIELD(org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) BulkAction(org.opensearch.action.bulk.BulkAction) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ExceptionUtil.getShardsFailure(org.opensearch.ad.util.ExceptionUtil.getShardsFailure) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) STOPPED_BY_FIELD(org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD) WriteRequest(org.opensearch.action.support.WriteRequest) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) ForwardADTaskAction(org.opensearch.ad.transport.ForwardADTaskAction) ActionListener(org.opensearch.action.ActionListener) DETECTOR_IS_RUNNING(org.opensearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) UpdateByQueryAction(org.opensearch.index.reindex.UpdateByQueryAction) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) TimeValue(org.opensearch.common.unit.TimeValue) RestHandlerUtils.createXContentParserFromRegistry(org.opensearch.ad.util.RestHandlerUtils.createXContentParserFromRegistry) SearchHit(org.opensearch.search.SearchHit) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) IS_LATEST_FIELD(org.opensearch.ad.model.ADTask.IS_LATEST_FIELD) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) ScoreMode(org.apache.lucene.search.join.ScoreMode) TransportService(org.opensearch.transport.TransportService) Logger(org.apache.logging.log4j.Logger) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) ADEntityTaskProfile(org.opensearch.ad.model.ADEntityTaskProfile) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) TermsQueryBuilder(org.opensearch.index.query.TermsQueryBuilder) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HC_DETECTOR_TASK_IS_UPDATING(org.opensearch.ad.constant.CommonErrorMessages.HC_DETECTOR_TASK_IS_UPDATING) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ADCancelTaskAction(org.opensearch.ad.transport.ADCancelTaskAction) BiConsumer(java.util.function.BiConsumer) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) QueryBuilders(org.opensearch.index.query.QueryBuilders) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) IOException(java.io.IOException) TotalHits(org.apache.lucene.search.TotalHits) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ChronoUnit(java.time.temporal.ChronoUnit) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) ExceptionUtil.getErrorMessage(org.opensearch.ad.util.ExceptionUtil.getErrorMessage) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) COORDINATING_NODE_FIELD(org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) MAX_OLD_AD_TASK_DOCS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS) ADBatchAnomalyResultAction(org.opensearch.ad.transport.ADBatchAnomalyResultAction) ToXContent(org.opensearch.common.xcontent.ToXContent) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) BulkRequest(org.opensearch.action.bulk.BulkRequest) ERROR_FIELD(org.opensearch.ad.model.ADTask.ERROR_FIELD) EXECUTION_START_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) XContentParser(org.opensearch.common.xcontent.XContentParser) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) DETECTOR_ID_FIELD(org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) REALTIME_TASK_TYPES(org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES) NestedQueryBuilder(org.opensearch.index.query.NestedQueryBuilder) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) DetectorProfile(org.opensearch.ad.model.DetectorProfile) ImmutableSet(com.google.common.collect.ImmutableSet) ESTIMATED_MINUTES_LEFT_FIELD(org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD) NOT_ENDED_STATES(org.opensearch.ad.model.ADTaskState.NOT_ENDED_STATES) ImmutableMap(com.google.common.collect.ImmutableMap) Script(org.opensearch.script.Script) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) ADTaskType(org.opensearch.ad.model.ADTaskType) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) LAST_UPDATE_TIME_FIELD(org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TASK_TYPE_FIELD(org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD) Objects(java.util.Objects) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ALL_HISTORICAL_TASK_TYPES(org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) TASK_ID_FIELD(org.opensearch.ad.model.AnomalyResult.TASK_ID_FIELD) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) HashMap(java.util.HashMap) SortOrder(org.opensearch.search.sort.SortOrder) ImmutableList(com.google.common.collect.ImmutableList) EXCEED_HISTORICAL_ANALYSIS_LIMIT(org.opensearch.ad.constant.CommonErrorMessages.EXCEED_HISTORICAL_ANALYSIS_LIMIT) ADCancelTaskRequest(org.opensearch.ad.transport.ADCancelTaskRequest) SearchResponse(org.opensearch.action.search.SearchResponse) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) Iterator(java.util.Iterator) ALL_AD_RESULTS_INDEX_PATTERN(org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN) Semaphore(java.util.concurrent.Semaphore) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) AD_USED_BATCH_TASK_SLOT_COUNT(org.opensearch.ad.stats.InternalStatNames.AD_USED_BATCH_TASK_SLOT_COUNT) Entity(org.opensearch.ad.model.Entity) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ADTaskProfileAction(org.opensearch.ad.transport.ADTaskProfileAction) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) ADTaskProfileRequest(org.opensearch.ad.transport.ADTaskProfileRequest) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString)

Aggregations

ADTaskProfile (org.opensearch.ad.model.ADTaskProfile)21 ADTask (org.opensearch.ad.model.ADTask)7 ArrayList (java.util.ArrayList)6 IOException (java.io.IOException)5 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)5 ImmutableList (com.google.common.collect.ImmutableList)4 List (java.util.List)4 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)4 Version (org.opensearch.Version)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 Instant (java.time.Instant)3 Map (java.util.Map)3 XContentParser (org.opensearch.common.xcontent.XContentParser)3 RandomCutForest (com.amazon.randomcutforest.RandomCutForest)2 ThresholdedRandomCutForest (com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest)2 ChronoUnit (java.time.temporal.ChronoUnit)2 Locale (java.util.Locale)2 Optional (java.util.Optional)2 ExecutorService (java.util.concurrent.ExecutorService)2 LinkedBlockingDeque (java.util.concurrent.LinkedBlockingDeque)2