Examples with DETECTION_STATE_INDEX org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX used on opensource projects

Example 11 with DETECTION_STATE_INDEX

use of org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX in project anomaly-detection by opensearch-project.

the class ADTaskManager method updateLatestFlagOfOldTasksAndCreateNewTask.

private void updateLatestFlagOfOldTasksAndCreateNewTask(AnomalyDetector detector, DetectionDateRange detectionDateRange, User user, ActionListener<AnomalyDetectorJobResponse> listener) {
    UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest();
    updateByQueryRequest.indices(DETECTION_STATE_INDEX);
    BoolQueryBuilder query = new BoolQueryBuilder();
    query.filter(new TermQueryBuilder(DETECTOR_ID_FIELD, detector.getDetectorId()));
    query.filter(new TermQueryBuilder(IS_LATEST_FIELD, true));
    // make sure we reset all latest task as false when user switch from single entity to HC, vice versa.
    query.filter(new TermsQueryBuilder(TASK_TYPE_FIELD, taskTypeToString(getADTaskTypes(detectionDateRange, true))));
    updateByQueryRequest.setQuery(query);
    updateByQueryRequest.setRefresh(true);
    String script = String.format(Locale.ROOT, "ctx._source.%s=%s;", IS_LATEST_FIELD, false);
    updateByQueryRequest.setScript(new Script(script));
    client.execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(r -> {
        List<BulkItemResponse.Failure> bulkFailures = r.getBulkFailures();
        if (bulkFailures.isEmpty()) {
            // Realtime AD coordinating node is chosen by job scheduler, we won't know it until realtime AD job
            // runs. Just set realtime AD coordinating node as null here, and AD job runner will reset correct
            // coordinating node once realtime job starts.
            // For historical analysis, this method will be called on coordinating node, so we can set coordinating
            // node as local node.
            String coordinatingNode = detectionDateRange == null ? null : clusterService.localNode().getId();
            createNewADTask(detector, detectionDateRange, user, coordinatingNode, listener);
        } else {
            logger.error("Failed to update old task's state for detector: {}, response: {} ", detector.getDetectorId(), r.toString());
            listener.onFailure(bulkFailures.get(0).getCause());
        }
    }, e -> {
        logger.error("Failed to reset old tasks as not latest for detector " + detector.getDetectorId(), e);
        listener.onFailure(e);
    }));
}

Example 12 with DETECTION_STATE_INDEX

use of org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX in project anomaly-detection by opensearch-project.

the class ADTaskManager method cleanChildTasksAndADResultsOfDeletedTask.

/**
 * Poll deleted detector task from cache and delete its child tasks and AD results.
 */
public void cleanChildTasksAndADResultsOfDeletedTask() {
    if (!adTaskCacheManager.hasDeletedDetectorTask()) {
        return;
    }
    threadPool.schedule(() -> {
        String taskId = adTaskCacheManager.pollDeletedDetectorTask();
        if (taskId == null) {
            return;
        }
        DeleteByQueryRequest deleteADResultsRequest = new DeleteByQueryRequest(ALL_AD_RESULTS_INDEX_PATTERN);
        deleteADResultsRequest.setQuery(new TermsQueryBuilder(TASK_ID_FIELD, taskId));
        client.execute(DeleteByQueryAction.INSTANCE, deleteADResultsRequest, ActionListener.wrap(res -> {
            logger.debug("Successfully deleted AD results of task " + taskId);
            DeleteByQueryRequest deleteChildTasksRequest = new DeleteByQueryRequest(DETECTION_STATE_INDEX);
            deleteChildTasksRequest.setQuery(new TermsQueryBuilder(PARENT_TASK_ID_FIELD, taskId));
            client.execute(DeleteByQueryAction.INSTANCE, deleteChildTasksRequest, ActionListener.wrap(r -> {
                logger.debug("Successfully deleted child tasks of task " + taskId);
                cleanChildTasksAndADResultsOfDeletedTask();
            }, e -> {
                logger.error("Failed to delete child tasks of task " + taskId, e);
            }));
        }, ex -> {
            logger.error("Failed to delete AD results for task " + taskId, ex);
        }));
    }, TimeValue.timeValueSeconds(DEFAULT_MAINTAIN_INTERVAL_IN_SECONDS), AD_BATCH_TASK_THREAD_POOL_NAME);
}

Example 13 with DETECTION_STATE_INDEX

use of org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX in project anomaly-detection by opensearch-project.

the class ADDataMigrator method createRealtimeADTask.

private void createRealtimeADTask(AnomalyDetectorJob job, String error, ConcurrentLinkedQueue<AnomalyDetectorJob> detectorJobs, boolean migrateAll) {
    client.get(new GetRequest(ANOMALY_DETECTORS_INDEX, job.getName()), ActionListener.wrap(r -> {
        if (r != null && r.isExists()) {
            try (XContentParser parser = createXContentParserFromRegistry(xContentRegistry, r.getSourceAsBytesRef())) {
                ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
                AnomalyDetector detector = AnomalyDetector.parse(parser, r.getId());
                ADTaskType taskType = detector.isMultientityDetector() ? ADTaskType.REALTIME_HC_DETECTOR : ADTaskType.REALTIME_SINGLE_ENTITY;
                Instant now = Instant.now();
                String userName = job.getUser() != null ? job.getUser().getName() : null;
                ADTask adTask = new ADTask.Builder().detectorId(detector.getDetectorId()).detector(detector).error(error).isLatest(true).taskType(taskType.name()).executionStartTime(now).taskProgress(0.0f).initProgress(0.0f).state(ADTaskState.CREATED.name()).lastUpdateTime(now).startedBy(userName).coordinatingNode(null).detectionDateRange(null).user(job.getUser()).build();
                IndexRequest indexRequest = new IndexRequest(DETECTION_STATE_INDEX).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).source(adTask.toXContent(XContentFactory.jsonBuilder(), XCONTENT_WITH_TYPE));
                client.index(indexRequest, ActionListener.wrap(indexResponse -> {
                    logger.info("Backfill realtime task successfully for detector {}", job.getName());
                    backfillRealtimeTask(detectorJobs, migrateAll);
                }, ex -> {
                    logger.error("Failed to backfill realtime task for detector " + job.getName(), ex);
                    backfillRealtimeTask(detectorJobs, migrateAll);
                }));
            } catch (IOException e) {
                logger.error("Fail to parse detector " + job.getName(), e);
                backfillRealtimeTask(detectorJobs, migrateAll);
            }
        } else {
            logger.error("Detector doesn't exist " + job.getName());
            backfillRealtimeTask(detectorJobs, migrateAll);
        }
    }, e -> {
        logger.error("Fail to get detector " + job.getName(), e);
        backfillRealtimeTask(detectorJobs, migrateAll);
    }));
}

Example 14 with DETECTION_STATE_INDEX

use of org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX in project anomaly-detection by opensearch-project.

the class ADTaskManagerTests method testDeleteADTasksWithException.

@SuppressWarnings("unchecked")
public void testDeleteADTasksWithException() {
    doAnswer(invocation -> {
        ActionListener<BulkByScrollResponse> actionListener = invocation.getArgument(2);
        actionListener.onFailure(new IndexNotFoundException(DETECTION_STATE_INDEX));
        return null;
    }).doAnswer(invocation -> {
        ActionListener<BulkByScrollResponse> actionListener = invocation.getArgument(2);
        actionListener.onFailure(new RuntimeException("test"));
        return null;
    }).when(client).execute(any(), any(), any());
    String detectorId = randomAlphaOfLength(5);
    AnomalyDetectorFunction function = mock(AnomalyDetectorFunction.class);
    ActionListener<DeleteResponse> listener = mock(ActionListener.class);
    adTaskManager.deleteADTasks(detectorId, function, listener);
    verify(function, times(1)).execute();
    verify(listener, never()).onFailure(any());
    adTaskManager.deleteADTasks(detectorId, function, listener);
    verify(function, times(1)).execute();
    verify(listener, times(1)).onFailure(any());
}