use of org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN in project anomaly-detection by opensearch-project.
the class SearchAnomalyResultTransportAction method doExecute.
@Override
protected void doExecute(Task task, SearchRequest request, ActionListener<SearchResponse> listener) {
String[] indices = request.indices();
if (indices == null || indices.length == 0) {
listener.onFailure(new IllegalArgumentException("No indices set in search request"));
return;
}
// Set query indices as default result indices, will check custom result indices permission and add
// custom indices which user has search permission later.
boolean onlyQueryCustomResultIndex = true;
for (String indexName : indices) {
// If only query custom result index, don't need to set ALL_AD_RESULTS_INDEX_PATTERN in search request
if (ALL_AD_RESULTS_INDEX_PATTERN.equals(indexName)) {
onlyQueryCustomResultIndex = false;
}
}
String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterService.state(), IndicesOptions.lenientExpandOpen(), indices);
// If concreteIndices is null or empty, don't throw exception. Detector list page will search both
// default and custom result indices to get anomaly of last 24 hours. If throw exception, detector
// list page will throw error and won't show any detector.
// If a cluster has no custom result indices, and some new non-custom-result-detector that hasn't
// finished one interval (where no default result index exists), then no result indices found. We
// will still search ".opendistro-anomaly-results*" (even these default indices don't exist) to
// return an empty SearchResponse. This search looks unnecessary, but this can make sure the
// detector list page show all detectors correctly. The other solution is to catch errors from
// frontend when search anomaly results to make sure frontend won't crash. Check this Github issue:
// https://github.com/opensearch-project/anomaly-detection-dashboards-plugin/issues/154
Set<String> customResultIndices = new HashSet<>();
if (concreteIndices != null) {
for (String index : concreteIndices) {
if (index.startsWith(CUSTOM_RESULT_INDEX_PREFIX)) {
customResultIndices.add(index);
}
}
}
// we should not search anymore. Just throw exception here.
if (onlyQueryCustomResultIndex && customResultIndices.size() == 0) {
listener.onFailure(new IllegalArgumentException("No custom result indices found"));
return;
}
if (customResultIndices.size() > 0) {
// Search both custom AD result index and default result index
String resultIndexAggName = "result_index";
SearchSourceBuilder searchResultIndexBuilder = new SearchSourceBuilder();
AggregationBuilder aggregation = new TermsAggregationBuilder(resultIndexAggName).field(AnomalyDetector.RESULT_INDEX_FIELD).size(MAX_DETECTOR_UPPER_LIMIT);
searchResultIndexBuilder.aggregation(aggregation).size(0);
SearchRequest searchResultIndex = new SearchRequest(AnomalyDetector.ANOMALY_DETECTORS_INDEX).source(searchResultIndexBuilder);
try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
// Search result indices of all detectors. User may create index with same prefix of custom result index
// which not used for AD, so we should avoid searching extra indices which not used by anomaly detectors.
// Variable used in lambda expression should be final or effectively final, so copy to a final boolean and
// use the final boolean in lambda below.
boolean finalOnlyQueryCustomResultIndex = onlyQueryCustomResultIndex;
client.search(searchResultIndex, ActionListener.wrap(allResultIndicesResponse -> {
Aggregations aggregations = allResultIndicesResponse.getAggregations();
StringTerms resultIndicesAgg = aggregations.get(resultIndexAggName);
List<StringTerms.Bucket> buckets = resultIndicesAgg.getBuckets();
Set<String> resultIndicesOfDetector = new HashSet<>();
if (buckets == null) {
searchHandler.search(request, listener);
return;
}
buckets.stream().forEach(b -> resultIndicesOfDetector.add(b.getKeyAsString()));
List<String> targetIndices = new ArrayList<>();
for (String index : customResultIndices) {
if (resultIndicesOfDetector.contains(index)) {
targetIndices.add(index);
}
}
if (targetIndices.size() == 0) {
// No custom result indices used by detectors, just search default result index
searchHandler.search(request, listener);
return;
}
MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
for (String index : targetIndices) {
multiSearchRequest.add(new SearchRequest(index).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(0)));
}
List<String> readableIndices = new ArrayList<>();
if (!finalOnlyQueryCustomResultIndex) {
readableIndices.add(ALL_AD_RESULTS_INDEX_PATTERN);
}
context.restore();
// Send multiple search to check which index a user has permission to read. If search all indices directly,
// search request will throw exception if user has no permission to search any index.
client.multiSearch(multiSearchRequest, ActionListener.wrap(multiSearchResponse -> {
MultiSearchResponse.Item[] responses = multiSearchResponse.getResponses();
for (int i = 0; i < responses.length; i++) {
MultiSearchResponse.Item item = responses[i];
String indexName = targetIndices.get(i);
if (item.getFailure() == null) {
readableIndices.add(indexName);
}
}
if (readableIndices.size() == 0) {
listener.onFailure(new IllegalArgumentException("No readable custom result indices found"));
return;
}
request.indices(readableIndices.toArray(new String[0]));
searchHandler.search(request, listener);
}, multiSearchException -> {
logger.error("Failed to search custom AD result indices", multiSearchException);
listener.onFailure(multiSearchException);
}));
}, e -> {
logger.error("Failed to search result indices for all detectors", e);
listener.onFailure(e);
}));
} catch (Exception e) {
logger.error(e);
listener.onFailure(e);
}
} else {
// onlyQueryCustomResultIndex is false in this branch
// Search only default result index
request.indices(ALL_AD_RESULTS_INDEX_PATTERN);
searchHandler.search(request, listener);
}
}
use of org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN in project anomaly-detection by opensearch-project.
the class ADTaskManager method deleteADResultOfDetector.
private void deleteADResultOfDetector(String detectorId) {
if (!deleteADResultWhenDeleteDetector) {
logger.info("Won't delete ad result for {} as delete AD result setting is disabled", detectorId);
return;
}
logger.info("Start to delete AD results of detector {}", detectorId);
DeleteByQueryRequest deleteADResultsRequest = new DeleteByQueryRequest(ALL_AD_RESULTS_INDEX_PATTERN);
deleteADResultsRequest.setQuery(new TermQueryBuilder(DETECTOR_ID_FIELD, detectorId));
client.execute(DeleteByQueryAction.INSTANCE, deleteADResultsRequest, ActionListener.wrap(response -> {
logger.debug("Successfully deleted AD results of detector " + detectorId);
}, exception -> {
logger.error("Failed to delete AD results of detector " + detectorId, exception);
adTaskCacheManager.addDeletedDetector(detectorId);
}));
}
use of org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN in project anomaly-detection by opensearch-project.
the class ADTaskManager method cleanChildTasksAndADResultsOfDeletedTask.
/**
* Poll deleted detector task from cache and delete its child tasks and AD results.
*/
public void cleanChildTasksAndADResultsOfDeletedTask() {
if (!adTaskCacheManager.hasDeletedDetectorTask()) {
return;
}
threadPool.schedule(() -> {
String taskId = adTaskCacheManager.pollDeletedDetectorTask();
if (taskId == null) {
return;
}
DeleteByQueryRequest deleteADResultsRequest = new DeleteByQueryRequest(ALL_AD_RESULTS_INDEX_PATTERN);
deleteADResultsRequest.setQuery(new TermsQueryBuilder(TASK_ID_FIELD, taskId));
client.execute(DeleteByQueryAction.INSTANCE, deleteADResultsRequest, ActionListener.wrap(res -> {
logger.debug("Successfully deleted AD results of task " + taskId);
DeleteByQueryRequest deleteChildTasksRequest = new DeleteByQueryRequest(DETECTION_STATE_INDEX);
deleteChildTasksRequest.setQuery(new TermsQueryBuilder(PARENT_TASK_ID_FIELD, taskId));
client.execute(DeleteByQueryAction.INSTANCE, deleteChildTasksRequest, ActionListener.wrap(r -> {
logger.debug("Successfully deleted child tasks of task " + taskId);
cleanChildTasksAndADResultsOfDeletedTask();
}, e -> {
logger.error("Failed to delete child tasks of task " + taskId, e);
}));
}, ex -> {
logger.error("Failed to delete AD results for task " + taskId, ex);
}));
}, TimeValue.timeValueSeconds(DEFAULT_MAINTAIN_INTERVAL_IN_SECONDS), AD_BATCH_TASK_THREAD_POOL_NAME);
}
use of org.opensearch.ad.indices.AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN in project anomaly-detection by opensearch-project.
the class RestDeleteAnomalyResultsAction method prepareRequest.
@Override
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
if (!EnabledSetting.isADPluginEnabled()) {
throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG);
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.parseXContent(request.contentOrSourceParamParser());
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest(ALL_AD_RESULTS_INDEX_PATTERN).setQuery(searchSourceBuilder.query()).setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN);
return channel -> client.execute(DeleteAnomalyResultsAction.INSTANCE, deleteRequest, ActionListener.wrap(r -> {
XContentBuilder contentBuilder = r.toXContent(channel.newBuilder().startObject(), ToXContent.EMPTY_PARAMS);
contentBuilder.endObject();
channel.sendResponse(new BytesRestResponse(RestStatus.OK, contentBuilder));
}, e -> {
try {
channel.sendResponse(new BytesRestResponse(channel, e));
} catch (IOException exception) {
logger.error("Failed to send back delete anomaly result exception result", exception);
}
}));
}
Aggregations