use of org.opensearch.ad.model.MergeableList in project anomaly-detection by opensearch-project.
the class AbstractAnomalyDetectorActionHandler method validateAnomalyDetectorFeatures.
/**
* Validate config/syntax, and runtime error of detector features
* @param detectorId detector id
* @param indexingDryRun if false, then will eventually index detector; true, skip indexing detector
* @throws IOException when fail to parse feature aggregation
*/
// TODO: move this method to util class so that it can be re-usable for more use cases
// https://github.com/opensearch-project/anomaly-detection/issues/39
protected void validateAnomalyDetectorFeatures(String detectorId, boolean indexingDryRun) throws IOException {
if (anomalyDetector != null && (anomalyDetector.getFeatureAttributes() == null || anomalyDetector.getFeatureAttributes().isEmpty())) {
checkADNameExists(detectorId, indexingDryRun);
return;
}
// checking configuration/syntax error of detector features
String error = RestHandlerUtils.checkAnomalyDetectorFeaturesSyntax(anomalyDetector, maxAnomalyFeatures);
if (StringUtils.isNotBlank(error)) {
if (indexingDryRun) {
listener.onFailure(new ADValidationException(error, DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.DETECTOR));
return;
}
listener.onFailure(new OpenSearchStatusException(error, RestStatus.BAD_REQUEST));
return;
}
// checking runtime error from feature query
ActionListener<MergeableList<Optional<double[]>>> validateFeatureQueriesListener = ActionListener.wrap(response -> {
checkADNameExists(detectorId, indexingDryRun);
}, exception -> {
listener.onFailure(new ADValidationException(exception.getMessage(), DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.DETECTOR));
});
MultiResponsesDelegateActionListener<MergeableList<Optional<double[]>>> multiFeatureQueriesResponseListener = new MultiResponsesDelegateActionListener<MergeableList<Optional<double[]>>>(validateFeatureQueriesListener, anomalyDetector.getFeatureAttributes().size(), String.format(Locale.ROOT, CommonErrorMessages.VALIDATION_FEATURE_FAILURE, anomalyDetector.getName()), false);
for (Feature feature : anomalyDetector.getFeatureAttributes()) {
SearchSourceBuilder ssb = new SearchSourceBuilder().size(1).query(QueryBuilders.matchAllQuery());
AggregatorFactories.Builder internalAgg = parseAggregators(feature.getAggregation().toString(), xContentRegistry, feature.getId());
ssb.aggregation(internalAgg.getAggregatorFactories().iterator().next());
SearchRequest searchRequest = new SearchRequest().indices(anomalyDetector.getIndices().toArray(new String[0])).source(ssb);
client.search(searchRequest, ActionListener.wrap(response -> {
Optional<double[]> aggFeatureResult = searchFeatureDao.parseResponse(response, Arrays.asList(feature.getId()));
if (aggFeatureResult.isPresent()) {
multiFeatureQueriesResponseListener.onResponse(new MergeableList<Optional<double[]>>(new ArrayList<Optional<double[]>>(Arrays.asList(aggFeatureResult))));
} else {
String errorMessage = CommonErrorMessages.FEATURE_WITH_EMPTY_DATA_MSG + feature.getName();
logger.error(errorMessage);
multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(errorMessage, RestStatus.BAD_REQUEST));
}
}, e -> {
String errorMessage;
if (isExceptionCausedByInvalidQuery(e)) {
errorMessage = CommonErrorMessages.FEATURE_WITH_INVALID_QUERY_MSG + feature.getName();
} else {
errorMessage = CommonErrorMessages.UNKNOWN_SEARCH_QUERY_EXCEPTION_MSG + feature.getName();
}
logger.error(errorMessage, e);
multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(errorMessage, RestStatus.BAD_REQUEST, e));
}));
}
}
use of org.opensearch.ad.model.MergeableList in project anomaly-detection by opensearch-project.
the class ModelValidationActionHandler method checkFeatureQueryDelegate.
private void checkFeatureQueryDelegate(long latestTime) throws IOException {
ActionListener<MergeableList<double[]>> validateFeatureQueriesListener = ActionListener.wrap(response -> {
windowDelayRecommendation(latestTime);
}, exception -> {
listener.onFailure(new ADValidationException(exception.getMessage(), DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
});
MultiResponsesDelegateActionListener<MergeableList<double[]>> multiFeatureQueriesResponseListener = new MultiResponsesDelegateActionListener<>(validateFeatureQueriesListener, anomalyDetector.getFeatureAttributes().size(), CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, false);
for (Feature feature : anomalyDetector.getFeatureAttributes()) {
AggregationBuilder aggregation = getBucketAggregation(latestTime, (IntervalTimeConfiguration) anomalyDetector.getDetectionInterval());
BoolQueryBuilder query = QueryBuilders.boolQuery().filter(anomalyDetector.getFilterQuery());
List<String> featureFields = ParseUtils.getFieldNamesForFeature(feature, xContentRegistry);
for (String featureField : featureFields) {
query.filter(QueryBuilders.existsQuery(featureField));
}
SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(query, aggregation);
SearchRequest searchRequest = new SearchRequest(anomalyDetector.getIndices().toArray(new String[0])).source(searchSourceBuilder);
client.search(searchRequest, ActionListener.wrap(response -> {
Histogram aggregate = checkBucketResultErrors(response);
if (aggregate == null) {
return;
}
double fullBucketRate = processBucketAggregationResults(aggregate);
if (fullBucketRate < CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) {
multiFeatureQueriesResponseListener.onFailure(new ADValidationException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
} else {
multiFeatureQueriesResponseListener.onResponse(new MergeableList<>(new ArrayList<>(Collections.singletonList(new double[] { fullBucketRate }))));
}
}, e -> {
logger.error(e);
multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, RestStatus.BAD_REQUEST, e));
}));
}
}
Aggregations