Search in sources :

Example 1 with MergeableList

use of org.opensearch.ad.model.MergeableList in project anomaly-detection by opensearch-project.

the class AbstractAnomalyDetectorActionHandler method validateAnomalyDetectorFeatures.

/**
 * Validate config/syntax, and runtime error of detector features
 * @param detectorId detector id
 * @param indexingDryRun if false, then will eventually index detector; true, skip indexing detector
 * @throws IOException when fail to parse feature aggregation
 */
// TODO: move this method to util class so that it can be re-usable for more use cases
// https://github.com/opensearch-project/anomaly-detection/issues/39
protected void validateAnomalyDetectorFeatures(String detectorId, boolean indexingDryRun) throws IOException {
    if (anomalyDetector != null && (anomalyDetector.getFeatureAttributes() == null || anomalyDetector.getFeatureAttributes().isEmpty())) {
        checkADNameExists(detectorId, indexingDryRun);
        return;
    }
    // checking configuration/syntax error of detector features
    String error = RestHandlerUtils.checkAnomalyDetectorFeaturesSyntax(anomalyDetector, maxAnomalyFeatures);
    if (StringUtils.isNotBlank(error)) {
        if (indexingDryRun) {
            listener.onFailure(new ADValidationException(error, DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.DETECTOR));
            return;
        }
        listener.onFailure(new OpenSearchStatusException(error, RestStatus.BAD_REQUEST));
        return;
    }
    // checking runtime error from feature query
    ActionListener<MergeableList<Optional<double[]>>> validateFeatureQueriesListener = ActionListener.wrap(response -> {
        checkADNameExists(detectorId, indexingDryRun);
    }, exception -> {
        listener.onFailure(new ADValidationException(exception.getMessage(), DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.DETECTOR));
    });
    MultiResponsesDelegateActionListener<MergeableList<Optional<double[]>>> multiFeatureQueriesResponseListener = new MultiResponsesDelegateActionListener<MergeableList<Optional<double[]>>>(validateFeatureQueriesListener, anomalyDetector.getFeatureAttributes().size(), String.format(Locale.ROOT, CommonErrorMessages.VALIDATION_FEATURE_FAILURE, anomalyDetector.getName()), false);
    for (Feature feature : anomalyDetector.getFeatureAttributes()) {
        SearchSourceBuilder ssb = new SearchSourceBuilder().size(1).query(QueryBuilders.matchAllQuery());
        AggregatorFactories.Builder internalAgg = parseAggregators(feature.getAggregation().toString(), xContentRegistry, feature.getId());
        ssb.aggregation(internalAgg.getAggregatorFactories().iterator().next());
        SearchRequest searchRequest = new SearchRequest().indices(anomalyDetector.getIndices().toArray(new String[0])).source(ssb);
        client.search(searchRequest, ActionListener.wrap(response -> {
            Optional<double[]> aggFeatureResult = searchFeatureDao.parseResponse(response, Arrays.asList(feature.getId()));
            if (aggFeatureResult.isPresent()) {
                multiFeatureQueriesResponseListener.onResponse(new MergeableList<Optional<double[]>>(new ArrayList<Optional<double[]>>(Arrays.asList(aggFeatureResult))));
            } else {
                String errorMessage = CommonErrorMessages.FEATURE_WITH_EMPTY_DATA_MSG + feature.getName();
                logger.error(errorMessage);
                multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(errorMessage, RestStatus.BAD_REQUEST));
            }
        }, e -> {
            String errorMessage;
            if (isExceptionCausedByInvalidQuery(e)) {
                errorMessage = CommonErrorMessages.FEATURE_WITH_INVALID_QUERY_MSG + feature.getName();
            } else {
                errorMessage = CommonErrorMessages.UNKNOWN_SEARCH_QUERY_EXCEPTION_MSG + feature.getName();
            }
            logger.error(errorMessage, e);
            multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(errorMessage, RestStatus.BAD_REQUEST, e));
        }));
    }
}
Also used : Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) ParseUtils.listEqualsWithoutConsideringOrder(org.opensearch.ad.util.ParseUtils.listEqualsWithoutConsideringOrder) IndexResponse(org.opensearch.action.index.IndexResponse) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) FAIL_TO_FIND_DETECTOR_MSG(org.opensearch.ad.constant.CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) XContentParser(org.opensearch.common.xcontent.XContentParser) WriteRequest(org.opensearch.action.support.WriteRequest) Locale(java.util.Locale) Map(java.util.Map) ParseUtils.parseAggregators(org.opensearch.ad.util.ParseUtils.parseAggregators) XContentFactory(org.opensearch.common.xcontent.XContentFactory) ActionListener(org.opensearch.action.ActionListener) ActionResponse(org.opensearch.action.ActionResponse) GetResponse(org.opensearch.action.get.GetResponse) Client(org.opensearch.client.Client) NumericSetting(org.opensearch.ad.settings.NumericSetting) TimeValue(org.opensearch.common.unit.TimeValue) Feature(org.opensearch.ad.model.Feature) Set(java.util.Set) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) GetFieldMappingsRequest(org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsRequest) ADTaskManager(org.opensearch.ad.task.ADTaskManager) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TransportService(org.opensearch.transport.TransportService) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) GetFieldMappingsAction(org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsAction) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) List(java.util.List) Logger(org.apache.logging.log4j.Logger) QueryBuilder(org.opensearch.index.query.QueryBuilder) GetFieldMappingsResponse(org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsResponse) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) Optional(java.util.Optional) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) RestHandlerUtils(org.opensearch.ad.util.RestHandlerUtils) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) AggregatorFactories(org.opensearch.search.aggregations.AggregatorFactories) IndicesOptions(org.opensearch.action.support.IndicesOptions) ArrayList(java.util.ArrayList) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HashSet(java.util.HashSet) IndexAnomalyDetectorResponse(org.opensearch.ad.transport.IndexAnomalyDetectorResponse) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) SearchRequest(org.opensearch.action.search.SearchRequest) SearchResponse(org.opensearch.action.search.SearchResponse) MergeableList(org.opensearch.ad.model.MergeableList) RestHandlerUtils.isExceptionCausedByInvalidQuery(org.opensearch.ad.util.RestHandlerUtils.isExceptionCausedByInvalidQuery) ValidationAspect(org.opensearch.ad.model.ValidationAspect) ValidateAnomalyDetectorResponse(org.opensearch.ad.transport.ValidateAnomalyDetectorResponse) HISTORICAL_DETECTOR_TASK_TYPES(org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES) QueryBuilders(org.opensearch.index.query.QueryBuilders) RestValidateAnomalyDetectorAction(org.opensearch.ad.rest.RestValidateAnomalyDetectorAction) CommonName(org.opensearch.ad.constant.CommonName) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) RestRequest(org.opensearch.rest.RestRequest) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) IOException(java.io.IOException) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) CreateIndexResponse(org.opensearch.action.admin.indices.create.CreateIndexResponse) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) User(org.opensearch.commons.authuser.User) DetectorValidationIssueType(org.opensearch.ad.model.DetectorValidationIssueType) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) Clock(java.time.Clock) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) SearchRequest(org.opensearch.action.search.SearchRequest) Optional(java.util.Optional) MergeableList(org.opensearch.ad.model.MergeableList) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) Feature(org.opensearch.ad.model.Feature) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) AggregatorFactories(org.opensearch.search.aggregations.AggregatorFactories) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Example 2 with MergeableList

use of org.opensearch.ad.model.MergeableList in project anomaly-detection by opensearch-project.

the class ModelValidationActionHandler method checkFeatureQueryDelegate.

private void checkFeatureQueryDelegate(long latestTime) throws IOException {
    ActionListener<MergeableList<double[]>> validateFeatureQueriesListener = ActionListener.wrap(response -> {
        windowDelayRecommendation(latestTime);
    }, exception -> {
        listener.onFailure(new ADValidationException(exception.getMessage(), DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
    });
    MultiResponsesDelegateActionListener<MergeableList<double[]>> multiFeatureQueriesResponseListener = new MultiResponsesDelegateActionListener<>(validateFeatureQueriesListener, anomalyDetector.getFeatureAttributes().size(), CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, false);
    for (Feature feature : anomalyDetector.getFeatureAttributes()) {
        AggregationBuilder aggregation = getBucketAggregation(latestTime, (IntervalTimeConfiguration) anomalyDetector.getDetectionInterval());
        BoolQueryBuilder query = QueryBuilders.boolQuery().filter(anomalyDetector.getFilterQuery());
        List<String> featureFields = ParseUtils.getFieldNamesForFeature(feature, xContentRegistry);
        for (String featureField : featureFields) {
            query.filter(QueryBuilders.existsQuery(featureField));
        }
        SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(query, aggregation);
        SearchRequest searchRequest = new SearchRequest(anomalyDetector.getIndices().toArray(new String[0])).source(searchSourceBuilder);
        client.search(searchRequest, ActionListener.wrap(response -> {
            Histogram aggregate = checkBucketResultErrors(response);
            if (aggregate == null) {
                return;
            }
            double fullBucketRate = processBucketAggregationResults(aggregate);
            if (fullBucketRate < CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) {
                multiFeatureQueriesResponseListener.onFailure(new ADValidationException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
            } else {
                multiFeatureQueriesResponseListener.onResponse(new MergeableList<>(new ArrayList<>(Collections.singletonList(new double[] { fullBucketRate }))));
            }
        }, e -> {
            logger.error(e);
            multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, RestStatus.BAD_REQUEST, e));
        }));
    }
}
Also used : TOP_VALIDATE_TIMEOUT_IN_MILLIS(org.opensearch.ad.settings.AnomalyDetectorSettings.TOP_VALIDATE_TIMEOUT_IN_MILLIS) INTERVAL_RECOMMENDATION_DECREASING_MULTIPLIER(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_RECOMMENDATION_DECREASING_MULTIPLIER) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) TimeConfiguration(org.opensearch.ad.model.TimeConfiguration) MAX_INTERVAL_REC_LENGTH_IN_MINUTES(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_INTERVAL_REC_LENGTH_IN_MINUTES) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) FieldSortBuilder(org.opensearch.search.sort.FieldSortBuilder) Locale(java.util.Locale) Duration(java.time.Duration) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) Feature(org.opensearch.ad.model.Feature) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation) CONFIG_BUCKET_MINIMUM_SUCCESS_RATE(org.opensearch.ad.settings.AnomalyDetectorSettings.CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) Instant(java.time.Instant) RestStatus(org.opensearch.rest.RestStatus) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) Collectors(java.util.stream.Collectors) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) List(java.util.List) Logger(org.apache.logging.log4j.Logger) QueryBuilder(org.opensearch.index.query.QueryBuilder) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) Optional(java.util.Optional) TermsValuesSourceBuilder(org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) MAX_TIMES_DECREASING_INTERVAL(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TIMES_DECREASING_INTERVAL) BucketOrder(org.opensearch.search.aggregations.BucketOrder) HashMap(java.util.HashMap) Aggregations(org.opensearch.search.aggregations.Aggregations) DateHistogramInterval(org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) INTERVAL_RECOMMENDATION_INCREASING_MULTIPLIER(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_RECOMMENDATION_INCREASING_MULTIPLIER) INTERVAL_BUCKET_MINIMUM_SUCCESS_RATE(org.opensearch.ad.settings.AnomalyDetectorSettings.INTERVAL_BUCKET_MINIMUM_SUCCESS_RATE) ArrayList(java.util.ArrayList) LongBounds(org.opensearch.search.aggregations.bucket.histogram.LongBounds) SortOrder(org.opensearch.search.sort.SortOrder) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) SearchRequest(org.opensearch.action.search.SearchRequest) SearchResponse(org.opensearch.action.search.SearchResponse) MergeableList(org.opensearch.ad.model.MergeableList) EndRunException(org.opensearch.ad.common.exception.EndRunException) ValidationAspect(org.opensearch.ad.model.ValidationAspect) ValidateAnomalyDetectorResponse(org.opensearch.ad.transport.ValidateAnomalyDetectorResponse) QueryBuilders(org.opensearch.index.query.QueryBuilders) RangeQueryBuilder(org.opensearch.index.query.RangeQueryBuilder) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) IOException(java.io.IOException) PipelineAggregatorBuilders(org.opensearch.search.aggregations.PipelineAggregatorBuilders) CompositeAggregation(org.opensearch.search.aggregations.bucket.composite.CompositeAggregation) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) DetectorValidationIssueType(org.opensearch.ad.model.DetectorValidationIssueType) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) Clock(java.time.Clock) Comparator(java.util.Comparator) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) LogManager(org.apache.logging.log4j.LogManager) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) ParseUtils(org.opensearch.ad.util.ParseUtils) SearchRequest(org.opensearch.action.search.SearchRequest) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) MergeableList(org.opensearch.ad.model.MergeableList) MultiResponsesDelegateActionListener(org.opensearch.ad.util.MultiResponsesDelegateActionListener) Feature(org.opensearch.ad.model.Feature) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) ADValidationException(org.opensearch.ad.common.exception.ADValidationException) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Aggregations

IOException (java.io.IOException)2 Clock (java.time.Clock)2 Instant (java.time.Instant)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Locale (java.util.Locale)2 Map (java.util.Map)2 Optional (java.util.Optional)2 Collectors (java.util.stream.Collectors)2 LogManager (org.apache.logging.log4j.LogManager)2 Logger (org.apache.logging.log4j.Logger)2 OpenSearchStatusException (org.opensearch.OpenSearchStatusException)2 ActionListener (org.opensearch.action.ActionListener)2 SearchRequest (org.opensearch.action.search.SearchRequest)2 SearchResponse (org.opensearch.action.search.SearchResponse)2 ADValidationException (org.opensearch.ad.common.exception.ADValidationException)2 CommonErrorMessages (org.opensearch.ad.constant.CommonErrorMessages)2 SearchFeatureDao (org.opensearch.ad.feature.SearchFeatureDao)2 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)2 DetectorValidationIssueType (org.opensearch.ad.model.DetectorValidationIssueType)2