Examples with AnomalyResult org.opensearch.ad.model.AnomalyResult used on opensource projects

Example 16 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class TestHelpers method randomAnomalyDetectResult.

public static AnomalyResult randomAnomalyDetectResult(double score, String error, String taskId, boolean withUser) {
    User user = withUser ? randomUser() : null;
    List<DataByFeatureId> relavantAttribution = new ArrayList<DataByFeatureId>();
    relavantAttribution.add(new DataByFeatureId(randomAlphaOfLength(5), randomDoubleBetween(0, 1.0, true)));
    relavantAttribution.add(new DataByFeatureId(randomAlphaOfLength(5), randomDoubleBetween(0, 1.0, true)));
    List<DataByFeatureId> pastValues = new ArrayList<DataByFeatureId>();
    pastValues.add(new DataByFeatureId(randomAlphaOfLength(5), randomDouble()));
    pastValues.add(new DataByFeatureId(randomAlphaOfLength(5), randomDouble()));
    List<ExpectedValueList> expectedValuesList = new ArrayList<ExpectedValueList>();
    List<DataByFeatureId> expectedValues = new ArrayList<DataByFeatureId>();
    expectedValues.add(new DataByFeatureId(randomAlphaOfLength(5), randomDouble()));
    expectedValues.add(new DataByFeatureId(randomAlphaOfLength(5), randomDouble()));
    expectedValuesList.add(new ExpectedValueList(randomDoubleBetween(0, 1.0, true), expectedValues));
    return new AnomalyResult(randomAlphaOfLength(5), taskId, score, randomDouble(), randomDouble(), ImmutableList.of(randomFeatureData(), randomFeatureData()), Instant.now().truncatedTo(ChronoUnit.SECONDS), Instant.now().truncatedTo(ChronoUnit.SECONDS), Instant.now().truncatedTo(ChronoUnit.SECONDS), Instant.now().truncatedTo(ChronoUnit.SECONDS), error, null, user, CommonValue.NO_SCHEMA_VERSION, null, Instant.now().truncatedTo(ChronoUnit.SECONDS), relavantAttribution, pastValues, expectedValuesList, randomDoubleBetween(1.1, 10.0, true));
}

Example 17 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class AnomalyResultHandlerTests method savingFailureTemplate.

/**
 * Template to test exponential backoff retry during saving anomaly result.
 *
 * @param throwOpenSearchRejectedExecutionException whether to throw
 *                                          OpenSearchRejectedExecutionException in the
 *                                          client::index mock or not
 * @param latchCount                        used for coordinating. Equal to
 *                                          number of expected retries plus 1.
 * @throws InterruptedException if thread execution is interrupted
 * @throws IOException          if IO failures
 */
@SuppressWarnings("unchecked")
private void savingFailureTemplate(boolean throwOpenSearchRejectedExecutionException, int latchCount, boolean adResultIndexExists) throws InterruptedException, IOException {
    setUpSavingAnomalyResultIndex(adResultIndexExists);
    final CountDownLatch backoffLatch = new CountDownLatch(latchCount);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assertTrue(String.format("The size of args is %d.  Its content is %s", args.length, Arrays.toString(args)), args.length >= 2);
        IndexRequest request = invocation.getArgument(0);
        ActionListener<IndexResponse> listener = invocation.getArgument(1);
        assertTrue(request != null && listener != null);
        if (throwOpenSearchRejectedExecutionException) {
            listener.onFailure(new OpenSearchRejectedExecutionException(""));
        } else {
            listener.onFailure(new IllegalArgumentException());
        }
        backoffLatch.countDown();
        return null;
    }).when(client).index(any(IndexRequest.class), ArgumentMatchers.<ActionListener<IndexResponse>>any());
    Settings backoffSettings = Settings.builder().put("plugins.anomaly_detection.max_retry_for_backoff", 2).put("plugins.anomaly_detection.backoff_initial_delay", TimeValue.timeValueMillis(1)).build();
    AnomalyIndexHandler<AnomalyResult> handler = new AnomalyIndexHandler<AnomalyResult>(client, backoffSettings, threadPool, CommonName.ANOMALY_RESULT_INDEX_ALIAS, anomalyDetectionIndices, clientUtil, indexUtil, clusterService);
    handler.index(TestHelpers.randomAnomalyDetectResult(), detectorId, null);
    backoffLatch.await(1, TimeUnit.MINUTES);
}

Example 18 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunner method indexAnomalyResultException.

private void indexAnomalyResultException(AnomalyDetectorJob jobParameter, LockService lockService, LockModel lock, Instant detectionStartTime, Instant executionStartTime, String errorMessage, boolean releaseLock, String taskState) {
    String detectorId = jobParameter.getName();
    try {
        IntervalTimeConfiguration windowDelay = (IntervalTimeConfiguration) jobParameter.getWindowDelay();
        Instant dataStartTime = detectionStartTime.minus(windowDelay.getInterval(), windowDelay.getUnit());
        Instant dataEndTime = executionStartTime.minus(windowDelay.getInterval(), windowDelay.getUnit());
        User user = jobParameter.getUser();
        AnomalyResult anomalyResult = new AnomalyResult(detectorId, // no task id
        null, new ArrayList<FeatureData>(), dataStartTime, dataEndTime, executionStartTime, Instant.now(), errorMessage, // single-stream detectors have no entity
        null, user, anomalyDetectionIndices.getSchemaVersion(ADIndex.RESULT), // no model id
        null);
        String resultIndex = jobParameter.getResultIndex();
        if (resultIndex != null && !anomalyDetectionIndices.doesIndexExist(resultIndex)) {
            // Set result index as null, will write exception to default result index.
            anomalyResultHandler.index(anomalyResult, detectorId, null);
        } else {
            anomalyResultHandler.index(anomalyResult, detectorId, resultIndex);
        }
        updateLatestRealtimeTask(detectorId, taskState, null, null, errorMessage);
    } catch (Exception e) {
        log.error("Failed to index anomaly result for " + detectorId, e);
    } finally {
        if (releaseLock) {
            releaseLock(jobParameter, lockService, lock);
        }
    }
}

Example 19 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class ADResultBulkTransportAction method doExecute.

@Override
protected void doExecute(Task task, ADResultBulkRequest request, ActionListener<ADResultBulkResponse> listener) {
    // Concurrent indexing memory limit = 10% of heap
    // indexing pressure = indexing bytes / indexing limit
    // Write all until index pressure (global indexing memory pressure) is less than 80% of 10% of heap. Otherwise, index
    // all non-zero anomaly grade index requests and index zero anomaly grade index requests with probability (1 - index pressure).
    long totalBytes = indexingPressure.getCurrentCombinedCoordinatingAndPrimaryBytes() + indexingPressure.getCurrentReplicaBytes();
    float indexingPressurePercent = (float) totalBytes / primaryAndCoordinatingLimits;
    List<ResultWriteRequest> results = request.getAnomalyResults();
    if (results == null || results.size() < 1) {
        listener.onResponse(new ADResultBulkResponse());
    }
    BulkRequest bulkRequest = new BulkRequest();
    if (indexingPressurePercent <= softLimit) {
        for (ResultWriteRequest resultWriteRequest : results) {
            addResult(bulkRequest, resultWriteRequest.getResult(), resultWriteRequest.getResultIndex());
        }
    } else if (indexingPressurePercent <= hardLimit) {
        // exceed soft limit (60%) but smaller than hard limit (90%)
        float acceptProbability = 1 - indexingPressurePercent;
        for (ResultWriteRequest resultWriteRequest : results) {
            AnomalyResult result = resultWriteRequest.getResult();
            if (result.isHighPriority() || random.nextFloat() < acceptProbability) {
                addResult(bulkRequest, result, resultWriteRequest.getResultIndex());
            }
        }
    } else {
        // if exceeding hard limit, only index non-zero grade or error result
        for (ResultWriteRequest resultWriteRequest : results) {
            AnomalyResult result = resultWriteRequest.getResult();
            if (result.isHighPriority()) {
                addResult(bulkRequest, result, resultWriteRequest.getResultIndex());
            }
        }
    }
    if (bulkRequest.numberOfActions() > 0) {
        client.execute(BulkAction.INSTANCE, bulkRequest, ActionListener.wrap(bulkResponse -> {
            List<IndexRequest> failedRequests = BulkUtil.getFailedIndexRequest(bulkRequest, bulkResponse);
            listener.onResponse(new ADResultBulkResponse(failedRequests));
        }, e -> {
            LOG.error("Failed to bulk index AD result", e);
            listener.onFailure(e);
        }));
    } else {
        listener.onResponse(new ADResultBulkResponse());
    }
}

Example 20 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class AnomalyDetectorPlugin method getRestHandlers.

@Override
public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
    AnomalyIndexHandler<AnomalyResult> anomalyResultHandler = new AnomalyIndexHandler<AnomalyResult>(client, settings, threadPool, CommonName.ANOMALY_RESULT_INDEX_ALIAS, anomalyDetectionIndices, this.clientUtil, this.indexUtils, clusterService);
    AnomalyDetectorJobRunner jobRunner = AnomalyDetectorJobRunner.getJobRunnerInstance();
    jobRunner.setClient(client);
    jobRunner.setThreadPool(threadPool);
    jobRunner.setAnomalyResultHandler(anomalyResultHandler);
    jobRunner.setSettings(settings);
    jobRunner.setAnomalyDetectionIndices(anomalyDetectionIndices);
    jobRunner.setNodeFilter(nodeFilter);
    jobRunner.setAdTaskManager(adTaskManager);
    RestGetAnomalyDetectorAction restGetAnomalyDetectorAction = new RestGetAnomalyDetectorAction();
    RestIndexAnomalyDetectorAction restIndexAnomalyDetectorAction = new RestIndexAnomalyDetectorAction(settings, clusterService);
    RestSearchAnomalyDetectorAction searchAnomalyDetectorAction = new RestSearchAnomalyDetectorAction();
    RestSearchAnomalyResultAction searchAnomalyResultAction = new RestSearchAnomalyResultAction();
    RestSearchADTasksAction searchADTasksAction = new RestSearchADTasksAction();
    RestDeleteAnomalyDetectorAction deleteAnomalyDetectorAction = new RestDeleteAnomalyDetectorAction();
    RestExecuteAnomalyDetectorAction executeAnomalyDetectorAction = new RestExecuteAnomalyDetectorAction(settings, clusterService);
    RestStatsAnomalyDetectorAction statsAnomalyDetectorAction = new RestStatsAnomalyDetectorAction(adStats, this.nodeFilter);
    RestAnomalyDetectorJobAction anomalyDetectorJobAction = new RestAnomalyDetectorJobAction(settings, clusterService);
    RestSearchAnomalyDetectorInfoAction searchAnomalyDetectorInfoAction = new RestSearchAnomalyDetectorInfoAction();
    RestPreviewAnomalyDetectorAction previewAnomalyDetectorAction = new RestPreviewAnomalyDetectorAction();
    RestDeleteAnomalyResultsAction deleteAnomalyResultsAction = new RestDeleteAnomalyResultsAction();
    RestSearchTopAnomalyResultAction searchTopAnomalyResultAction = new RestSearchTopAnomalyResultAction();
    RestValidateAnomalyDetectorAction validateAnomalyDetectorAction = new RestValidateAnomalyDetectorAction(settings, clusterService);
    return ImmutableList.of(restGetAnomalyDetectorAction, restIndexAnomalyDetectorAction, searchAnomalyDetectorAction, searchAnomalyResultAction, searchADTasksAction, deleteAnomalyDetectorAction, executeAnomalyDetectorAction, anomalyDetectorJobAction, statsAnomalyDetectorAction, searchAnomalyDetectorInfoAction, previewAnomalyDetectorAction, deleteAnomalyResultsAction, searchTopAnomalyResultAction, validateAnomalyDetectorAction);
}