Search in sources :

Example 11 with IntervalTimeConfiguration

use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.

the class EntityColdStarterTests method setUp.

@SuppressWarnings("unchecked")
@Override
public void setUp() throws Exception {
    super.setUp();
    numMinSamples = AnomalyDetectorSettings.NUM_MIN_SAMPLES;
    clock = mock(Clock.class);
    when(clock.instant()).thenReturn(Instant.now());
    threadPool = mock(ThreadPool.class);
    setUpADThreadPool(threadPool);
    settings = Settings.EMPTY;
    Client client = mock(Client.class);
    clientUtil = mock(ClientUtil.class);
    detector = TestHelpers.AnomalyDetectorBuilder.newInstance().setDetectionInterval(new IntervalTimeConfiguration(1, ChronoUnit.MINUTES)).setCategoryFields(ImmutableList.of(randomAlphaOfLength(5))).build();
    job = TestHelpers.randomAnomalyDetectorJob(true, Instant.ofEpochMilli(1602401500000L), null);
    doAnswer(invocation -> {
        GetRequest request = invocation.getArgument(0);
        ActionListener<GetResponse> listener = invocation.getArgument(2);
        if (request.index().equals(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(job, detectorId, AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX));
        } else {
            listener.onResponse(TestHelpers.createGetResponse(detector, detectorId, AnomalyDetector.ANOMALY_DETECTORS_INDEX));
        }
        return null;
    }).when(clientUtil).asyncRequest(any(GetRequest.class), any(), any(ActionListener.class));
    Set<Setting<?>> nodestateSetting = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    nodestateSetting.add(MAX_RETRY_FOR_UNRESPONSIVE_NODE);
    nodestateSetting.add(BACKOFF_MINUTES);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, nodestateSetting);
    DiscoveryNode discoveryNode = new DiscoveryNode("node1", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
    ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, clusterSettings);
    stateManager = new NodeStateManager(client, xContentRegistry(), settings, clientUtil, clock, AnomalyDetectorSettings.HOURLY_MAINTENANCE, clusterService);
    SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = new IntegerSensitiveSingleFeatureLinearUniformInterpolator();
    interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator);
    searchFeatureDao = mock(SearchFeatureDao.class);
    checkpoint = mock(CheckpointDao.class);
    featureManager = new FeatureManager(searchFeatureDao, interpolator, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.TRAIN_SAMPLE_TIME_RANGE_IN_HOURS, AnomalyDetectorSettings.MIN_TRAIN_SAMPLES, AnomalyDetectorSettings.MAX_SHINGLE_PROPORTION_MISSING, AnomalyDetectorSettings.MAX_IMPUTATION_NEIGHBOR_DISTANCE, AnomalyDetectorSettings.PREVIEW_SAMPLE_RATE, AnomalyDetectorSettings.MAX_PREVIEW_SAMPLES, AnomalyDetectorSettings.HOURLY_MAINTENANCE, threadPool, AnomalyDetectorPlugin.AD_THREAD_POOL_NAME);
    checkpointWriteQueue = mock(CheckpointWriteWorker.class);
    rcfSeed = 2051L;
    entityColdStarter = new EntityColdStarter(clock, threadPool, stateManager, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.TIME_DECAY, numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, interpolator, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, settings, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue, rcfSeed, AnomalyDetectorSettings.MAX_COLD_START_ROUNDS);
    detectorId = "123";
    modelId = "123_entity_abc";
    entityName = "abc";
    priority = 0.3f;
    entity = Entity.createSingleAttributeEntity("field", entityName);
    released = new AtomicBoolean();
    inProgressLatch = new CountDownLatch(1);
    releaseSemaphore = () -> {
        released.set(true);
        inProgressLatch.countDown();
    };
    listener = ActionListener.wrap(releaseSemaphore);
    modelManager = new ModelManager(mock(CheckpointDao.class), mock(Clock.class), AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.TIME_DECAY, AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, AnomalyDetectorSettings.MIN_PREVIEW_SIZE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, entityColdStarter, mock(FeatureManager.class), mock(MemoryTracker.class));
}
Also used : DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) ClusterSettings(org.opensearch.common.settings.ClusterSettings) ClientUtil(org.opensearch.ad.util.ClientUtil) ThreadPool(org.opensearch.threadpool.ThreadPool) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) Clock(java.time.Clock) NodeStateManager(org.opensearch.ad.NodeStateManager) IntegerSensitiveSingleFeatureLinearUniformInterpolator(org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator) GetRequest(org.opensearch.action.get.GetRequest) SingleFeatureLinearUniformInterpolator(org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator) IntegerSensitiveSingleFeatureLinearUniformInterpolator(org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator) LinearUniformInterpolator(org.opensearch.ad.dataprocessor.LinearUniformInterpolator) Client(org.opensearch.client.Client) FeatureManager(org.opensearch.ad.feature.FeatureManager) HashSet(java.util.HashSet) Setting(org.opensearch.common.settings.Setting) CountDownLatch(java.util.concurrent.CountDownLatch) GetResponse(org.opensearch.action.get.GetResponse) SingleFeatureLinearUniformInterpolator(org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator) IntegerSensitiveSingleFeatureLinearUniformInterpolator(org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator) CheckpointWriteWorker(org.opensearch.ad.ratelimit.CheckpointWriteWorker) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterService(org.opensearch.cluster.service.ClusterService) ActionListener(org.opensearch.action.ActionListener)

Example 12 with IntervalTimeConfiguration

use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.

the class ADBatchTaskRunner method getDateRangeOfSourceData.

private void getDateRangeOfSourceData(ADTask adTask, BiConsumer<Long, Long> consumer, ActionListener<String> internalListener) {
    String taskId = adTask.getTaskId();
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().aggregation(AggregationBuilders.min(AGG_NAME_MIN_TIME).field(adTask.getDetector().getTimeField())).aggregation(AggregationBuilders.max(AGG_NAME_MAX_TIME).field(adTask.getDetector().getTimeField())).size(0);
    if (adTask.getEntity() != null && adTask.getEntity().getAttributes().size() > 0) {
        BoolQueryBuilder query = new BoolQueryBuilder();
        adTask.getEntity().getAttributes().entrySet().forEach(entity -> query.filter(new TermQueryBuilder(entity.getKey(), entity.getValue())));
        searchSourceBuilder.query(query);
    }
    SearchRequest request = new SearchRequest().indices(adTask.getDetector().getIndices().toArray(new String[0])).source(searchSourceBuilder);
    client.search(request, ActionListener.wrap(r -> {
        InternalMin minAgg = r.getAggregations().get(AGG_NAME_MIN_TIME);
        InternalMax maxAgg = r.getAggregations().get(AGG_NAME_MAX_TIME);
        double minValue = minAgg.getValue();
        double maxValue = maxAgg.getValue();
        // If time field not exist or there is no value, will return infinity value
        if (minValue == Double.POSITIVE_INFINITY) {
            internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the time field"));
            return;
        }
        long interval = ((IntervalTimeConfiguration) adTask.getDetector().getDetectionInterval()).toDuration().toMillis();
        DetectionDateRange detectionDateRange = adTask.getDetectionDateRange();
        long dataStartTime = detectionDateRange.getStartTime().toEpochMilli();
        long dataEndTime = detectionDateRange.getEndTime().toEpochMilli();
        long minDate = (long) minValue;
        long maxDate = (long) maxValue;
        if (minDate >= dataEndTime || maxDate <= dataStartTime) {
            internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the detection date range"));
            return;
        }
        if (minDate > dataStartTime) {
            dataStartTime = minDate;
        }
        if (maxDate < dataEndTime) {
            dataEndTime = maxDate;
        }
        // normalize start/end time to make it consistent with feature data agg result
        dataStartTime = dataStartTime - dataStartTime % interval;
        dataEndTime = dataEndTime - dataEndTime % interval;
        logger.debug("adjusted date range: start: {}, end: {}, taskId: {}", dataStartTime, dataEndTime, taskId);
        if ((dataEndTime - dataStartTime) < NUM_MIN_SAMPLES * interval) {
            internalListener.onFailure(new AnomalyDetectionException("There is not enough data to train model").countedInStats(false));
            return;
        }
        consumer.accept(dataStartTime, dataEndTime);
    }, e -> {
        internalListener.onFailure(e);
    }));
}
Also used : AnomalyResultBulkIndexHandler(org.opensearch.ad.transport.handler.AnomalyResultBulkIndexHandler) ModelManager(org.opensearch.ad.ml.ModelManager) HashRing(org.opensearch.ad.cluster.HashRing) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) ADIndex(org.opensearch.ad.indices.ADIndex) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) BATCH_TASK_PIECE_SIZE(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) TimeValue(org.opensearch.common.unit.TimeValue) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) TransportService(org.opensearch.transport.TransportService) RandomCutForest(com.amazon.randomcutforest.RandomCutForest) Logger(org.apache.logging.log4j.Logger) PriorityTracker(org.opensearch.ad.caching.PriorityTracker) ExceptionUtil(org.opensearch.ad.util.ExceptionUtil) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) CheckedRunnable(org.opensearch.common.CheckedRunnable) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) InjectSecurity(org.opensearch.commons.InjectSecurity) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) ADCircuitBreakerService(org.opensearch.ad.breaker.ADCircuitBreakerService) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) StringTerms(org.opensearch.search.aggregations.bucket.terms.StringTerms) ThreadPool(org.opensearch.threadpool.ThreadPool) EnabledSetting(org.opensearch.ad.settings.EnabledSetting) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) BiConsumer(java.util.function.BiConsumer) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) SinglePointFeatures(org.opensearch.ad.feature.SinglePointFeatures) FeatureManager(org.opensearch.ad.feature.FeatureManager) MAX_TOP_ENTITIES_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TOP_ENTITIES_FOR_HISTORICAL_ANALYSIS) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ClusterService(org.opensearch.cluster.service.ClusterService) StatNames(org.opensearch.ad.stats.StatNames) ParseUtils(org.opensearch.ad.util.ParseUtils) InternalMin(org.opensearch.search.aggregations.metrics.InternalMin) AD_EXECUTING_BATCH_TASK_COUNT(org.opensearch.ad.stats.StatNames.AD_EXECUTING_BATCH_TASK_COUNT) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADStats(org.opensearch.ad.stats.ADStats) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) ADBatchTaskRemoteExecutionAction(org.opensearch.ad.transport.ADBatchTaskRemoteExecutionAction) ADTaskType(org.opensearch.ad.model.ADTaskType) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) AnomalyResult(org.opensearch.ad.model.AnomalyResult) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) DEFAULT_JVM_HEAP_USAGE_THRESHOLD(org.opensearch.ad.breaker.MemoryCircuitBreaker.DEFAULT_JVM_HEAP_USAGE_THRESHOLD) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) AnomalyDescriptor(com.amazon.randomcutforest.parkservices.AnomalyDescriptor) ADTask(org.opensearch.ad.model.ADTask) FeatureData(org.opensearch.ad.model.FeatureData) HashMap(java.util.HashMap) Deque(java.util.Deque) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) CURRENT_PIECE_FIELD(org.opensearch.ad.model.ADTask.CURRENT_PIECE_FIELD) ImmutableList(com.google.common.collect.ImmutableList) ADBatchAnomalyResultResponse(org.opensearch.ad.transport.ADBatchAnomalyResultResponse) JVM_HEAP_USAGE(org.opensearch.ad.stats.InternalStatNames.JVM_HEAP_USAGE) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) AGG_NAME_MAX_TIME(org.opensearch.ad.constant.CommonName.AGG_NAME_MAX_TIME) RangeQueryBuilder(org.opensearch.index.query.RangeQueryBuilder) MAX_TOP_ENTITIES_LIMIT_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TOP_ENTITIES_LIMIT_FOR_HISTORICAL_ANALYSIS) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) WORKER_NODE_FIELD(org.opensearch.ad.model.ADTask.WORKER_NODE_FIELD) Entity(org.opensearch.ad.model.Entity) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) ThresholdedRandomCutForest(com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest) Clock(java.time.Clock) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) LogManager(org.apache.logging.log4j.LogManager) AGG_NAME_MIN_TIME(org.opensearch.ad.constant.CommonName.AGG_NAME_MIN_TIME) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) SearchRequest(org.opensearch.action.search.SearchRequest) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) InternalMin(org.opensearch.search.aggregations.metrics.InternalMin) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder)

Example 13 with IntervalTimeConfiguration

use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.

the class IndexAnomalyDetectorJobActionHandler method createJob.

private void createJob(AnomalyDetector detector) {
    try {
        IntervalTimeConfiguration interval = (IntervalTimeConfiguration) detector.getDetectionInterval();
        Schedule schedule = new IntervalSchedule(Instant.now(), (int) interval.getInterval(), interval.getUnit());
        Duration duration = Duration.of(interval.getInterval(), interval.getUnit());
        AnomalyDetectorJob job = new AnomalyDetectorJob(detector.getDetectorId(), schedule, detector.getWindowDelay(), true, Instant.now(), null, Instant.now(), duration.getSeconds(), detector.getUser(), detector.getResultIndex());
        getAnomalyDetectorJobForWrite(detector, job);
    } catch (Exception e) {
        String message = "Failed to parse anomaly detector job " + detectorId;
        logger.error(message, e);
        listener.onFailure(new OpenSearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR));
    }
}
Also used : IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) Schedule(org.opensearch.jobscheduler.spi.schedule.Schedule) IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Duration(java.time.Duration) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) IOException(java.io.IOException) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Example 14 with IntervalTimeConfiguration

use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.

the class ModelValidationActionHandler method checkDataFilterSparsity.

private void checkDataFilterSparsity(long latestTime) {
    AggregationBuilder aggregation = getBucketAggregation(latestTime, (IntervalTimeConfiguration) anomalyDetector.getDetectionInterval());
    BoolQueryBuilder query = QueryBuilders.boolQuery().filter(anomalyDetector.getFilterQuery());
    SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(query, aggregation);
    SearchRequest searchRequest = new SearchRequest(anomalyDetector.getIndices().toArray(new String[0])).source(searchSourceBuilder);
    client.search(searchRequest, ActionListener.wrap(response -> processDataFilterResults(response, latestTime), listener::onFailure));
}
Also used : TOP_VALIDATE_TIMEOUT_IN_MILLIS(org.opensearch.ad.settings.AnomalyDetectorSettings.TOP_VALIDATE_TIMEOUT_IN_MILLIS) INTERVAL_RECOMMENDATION_DECREASING_MULTIPLIER(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_RECOMMENDATION_DECREASING_MULTIPLIER) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) TimeConfiguration(org.opensearch.ad.model.TimeConfiguration) MAX_INTERVAL_REC_LENGTH_IN_MINUTES(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_INTERVAL_REC_LENGTH_IN_MINUTES) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) FieldSortBuilder(org.opensearch.search.sort.FieldSortBuilder) Locale(java.util.Locale) Duration(java.time.Duration) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) Feature(org.opensearch.ad.model.Feature) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation) CONFIG_BUCKET_MINIMUM_SUCCESS_RATE(org.opensearch.ad.settings.AnomalyDetectorSettings.CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) Collectors(java.util.stream.Collectors) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) List(java.util.List) Logger(org.apache.logging.log4j.Logger) QueryBuilder(org.opensearch.index.query.QueryBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) Optional(java.util.Optional) TermsValuesSourceBuilder(org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) MAX_TIMES_DECREASING_INTERVAL(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TIMES_DECREASING_INTERVAL) BucketOrder(org.opensearch.search.aggregations.BucketOrder) HashMap(java.util.HashMap) Aggregations(org.opensearch.search.aggregations.Aggregations) DateHistogramInterval(org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) INTERVAL_RECOMMENDATION_INCREASING_MULTIPLIER(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_RECOMMENDATION_INCREASING_MULTIPLIER) INTERVAL_BUCKET_MINIMUM_SUCCESS_RATE(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_BUCKET_MINIMUM_SUCCESS_RATE) ArrayList(java.util.ArrayList) LongBounds(org.opensearch.search.aggregations.bucket.histogram.LongBounds) SortOrder(org.opensearch.search.sort.SortOrder) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) SearchRequest(org.opensearch.action.search.SearchRequest) SearchResponse(org.opensearch.action.search.SearchResponse) MergeableList(org.opensearch.ad.model.MergeableList) EndRunException(org.opensearch.ad.common.exception.EndRunException) ValidationAspect(org.opensearch.ad.model.ValidationAspect) ValidateAnomalyDetectorResponse(org.opensearch.ad.transport.ValidateAnomalyDetectorResponse) QueryBuilders(org.opensearch.index.query.QueryBuilders) RangeQueryBuilder(org.opensearch.index.query.RangeQueryBuilder) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) IOException(java.io.IOException) PipelineAggregatorBuilders(org.opensearch.search.aggregations.PipelineAggregatorBuilders) CompositeAggregation(org.opensearch.search.aggregations.bucket.composite.CompositeAggregation) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) DetectorValidationIssueType(org.opensearch.ad.model.DetectorValidationIssueType) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) Clock(java.time.Clock) Comparator(java.util.Comparator) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) LogManager(org.apache.logging.log4j.LogManager) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) ParseUtils(org.opensearch.ad.util.ParseUtils) SearchRequest(org.opensearch.action.search.SearchRequest) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder)

Example 15 with IntervalTimeConfiguration

use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.

the class ModelValidationActionHandler method checkFeatureQueryDelegate.

private void checkFeatureQueryDelegate(long latestTime) throws IOException {
    ActionListener<MergeableList<double[]>> validateFeatureQueriesListener = ActionListener.wrap(response -> {
        windowDelayRecommendation(latestTime);
    }, exception -> {
        listener.onFailure(new ADValidationException(exception.getMessage(), DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
    });
    MultiResponsesDelegateActionListener<MergeableList<double[]>> multiFeatureQueriesResponseListener = new MultiResponsesDelegateActionListener<>(validateFeatureQueriesListener, anomalyDetector.getFeatureAttributes().size(), CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, false);
    for (Feature feature : anomalyDetector.getFeatureAttributes()) {
        AggregationBuilder aggregation = getBucketAggregation(latestTime, (IntervalTimeConfiguration) anomalyDetector.getDetectionInterval());
        BoolQueryBuilder query = QueryBuilders.boolQuery().filter(anomalyDetector.getFilterQuery());
        List<String> featureFields = ParseUtils.getFieldNamesForFeature(feature, xContentRegistry);
        for (String featureField : featureFields) {
            query.filter(QueryBuilders.existsQuery(featureField));
        }
        SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(query, aggregation);
        SearchRequest searchRequest = new SearchRequest(anomalyDetector.getIndices().toArray(new String[0])).source(searchSourceBuilder);
        client.search(searchRequest, ActionListener.wrap(response -> {
            Histogram aggregate = checkBucketResultErrors(response);
            if (aggregate == null) {
                return;
            }
            double fullBucketRate = processBucketAggregationResults(aggregate);
            if (fullBucketRate < CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) {
                multiFeatureQueriesResponseListener.onFailure(new ADValidationException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
            } else {
                multiFeatureQueriesResponseListener.onResponse(new MergeableList<>(new ArrayList<>(Collections.singletonList(new double[] { fullBucketRate }))));
            }
        }, e -> {
            logger.error(e);
            multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, RestStatus.BAD_REQUEST, e));
        }));
    }
}
Also used : TOP_VALIDATE_TIMEOUT_IN_MILLIS(org.opensearch.ad.settings.AnomalyDetectorSettings.TOP_VALIDATE_TIMEOUT_IN_MILLIS) INTERVAL_RECOMMENDATION_DECREASING_MULTIPLIER(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_RECOMMENDATION_DECREASING_MULTIPLIER) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) TimeConfiguration(org.opensearch.ad.model.TimeConfiguration) MAX_INTERVAL_REC_LENGTH_IN_MINUTES(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_INTERVAL_REC_LENGTH_IN_MINUTES) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) FieldSortBuilder(org.opensearch.search.sort.FieldSortBuilder) Locale(java.util.Locale) Duration(java.time.Duration) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) Feature(org.opensearch.ad.model.Feature) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation) CONFIG_BUCKET_MINIMUM_SUCCESS_RATE(org.opensearch.ad.settings.AnomalyDetectorSettings.CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) Collectors(java.util.stream.Collectors) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) List(java.util.List) Logger(org.apache.logging.log4j.Logger) QueryBuilder(org.opensearch.index.query.QueryBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) Optional(java.util.Optional) TermsValuesSourceBuilder(org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) MAX_TIMES_DECREASING_INTERVAL(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TIMES_DECREASING_INTERVAL) BucketOrder(org.opensearch.search.aggregations.BucketOrder) HashMap(java.util.HashMap) Aggregations(org.opensearch.search.aggregations.Aggregations) DateHistogramInterval(org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) INTERVAL_RECOMMENDATION_INCREASING_MULTIPLIER(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_RECOMMENDATION_INCREASING_MULTIPLIER) INTERVAL_BUCKET_MINIMUM_SUCCESS_RATE(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_BUCKET_MINIMUM_SUCCESS_RATE) ArrayList(java.util.ArrayList) LongBounds(org.opensearch.search.aggregations.bucket.histogram.LongBounds) SortOrder(org.opensearch.search.sort.SortOrder) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) SearchRequest(org.opensearch.action.search.SearchRequest) SearchResponse(org.opensearch.action.search.SearchResponse) MergeableList(org.opensearch.ad.model.MergeableList) EndRunException(org.opensearch.ad.common.exception.EndRunException) ValidationAspect(org.opensearch.ad.model.ValidationAspect) ValidateAnomalyDetectorResponse(org.opensearch.ad.transport.ValidateAnomalyDetectorResponse) QueryBuilders(org.opensearch.index.query.QueryBuilders) RangeQueryBuilder(org.opensearch.index.query.RangeQueryBuilder) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) IOException(java.io.IOException) PipelineAggregatorBuilders(org.opensearch.search.aggregations.PipelineAggregatorBuilders) CompositeAggregation(org.opensearch.search.aggregations.bucket.composite.CompositeAggregation) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) DetectorValidationIssueType(org.opensearch.ad.model.DetectorValidationIssueType) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) Clock(java.time.Clock) Comparator(java.util.Comparator) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) LogManager(org.apache.logging.log4j.LogManager) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) ParseUtils(org.opensearch.ad.util.ParseUtils) SearchRequest(org.opensearch.action.search.SearchRequest) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) MergeableList(org.opensearch.ad.model.MergeableList) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) Feature(org.opensearch.ad.model.Feature) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Aggregations

IntervalTimeConfiguration (org.opensearch.ad.model.IntervalTimeConfiguration)27 ActionListener (org.opensearch.action.ActionListener)17 ArrayList (java.util.ArrayList)12 List (java.util.List)11 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)11 IOException (java.io.IOException)10 Optional (java.util.Optional)10 EndRunException (org.opensearch.ad.common.exception.EndRunException)10 Instant (java.time.Instant)9 SearchRequest (org.opensearch.action.search.SearchRequest)9 Clock (java.time.Clock)8 Map (java.util.Map)8 CommonErrorMessages (org.opensearch.ad.constant.CommonErrorMessages)8 HashMap (java.util.HashMap)7 Collectors (java.util.stream.Collectors)7 LogManager (org.apache.logging.log4j.LogManager)7 Logger (org.apache.logging.log4j.Logger)7 GetRequest (org.opensearch.action.get.GetRequest)7 GetResponse (org.opensearch.action.get.GetResponse)7 SearchFeatureDao (org.opensearch.ad.feature.SearchFeatureDao)7