Examples with AnomalyResult org.opensearch.ad.model.AnomalyResult used on opensource projects

Example 11 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class CheckpointReadWorker method onGetDetector.

private ActionListener<Optional<AnomalyDetector>> onGetDetector(EntityFeatureRequest origRequest, int index, String detectorId, List<EntityFeatureRequest> toProcess, Map<String, MultiGetItemResponse> successfulRequests, Set<String> retryableRequests, Optional<Entry<EntityModel, Instant>> checkpoint, Entity entity, String modelId) {
    return ActionListener.wrap(detectorOptional -> {
        if (false == detectorOptional.isPresent()) {
            LOG.warn(new ParameterizedMessage("AnomalyDetector [{}] is not available.", detectorId));
            processCheckpointIteration(index + 1, toProcess, successfulRequests, retryableRequests);
            return;
        }
        AnomalyDetector detector = detectorOptional.get();
        ModelState<EntityModel> modelState = modelManager.processEntityCheckpoint(checkpoint, entity, modelId, detectorId, detector.getShingleSize());
        EntityModel entityModel = modelState.getModel();
        ThresholdingResult result = null;
        if (entityModel.getTrcf().isPresent()) {
            result = modelManager.score(origRequest.getCurrentFeature(), modelId, modelState);
        } else {
            entityModel.addSample(origRequest.getCurrentFeature());
        }
        if (result != null && result.getRcfScore() > 0) {
            AnomalyResult resultToSave = result.toAnomalyResult(detector, Instant.ofEpochMilli(origRequest.getDataStartTimeMillis()), Instant.ofEpochMilli(origRequest.getDataStartTimeMillis() + detector.getDetectorIntervalInMilliseconds()), Instant.now(), Instant.now(), ParseUtils.getFeatureData(origRequest.getCurrentFeature(), detector), entity, indexUtil.getSchemaVersion(ADIndex.RESULT), modelId, null, null);
            resultWriteQueue.put(new ResultWriteRequest(origRequest.getExpirationEpochMs(), detectorId, result.getGrade() > 0 ? RequestPriority.HIGH : RequestPriority.MEDIUM, resultToSave, detector.getResultIndex()));
        }
        // try to load to cache
        boolean loaded = cacheProvider.get().hostIfPossible(detector, modelState);
        if (false == loaded) {
            // not in memory. Maybe cold entities or some other entities
            // have filled the slot while waiting for loading checkpoints.
            checkpointWriteQueue.write(modelState, true, RequestPriority.LOW);
        }
        processCheckpointIteration(index + 1, toProcess, successfulRequests, retryableRequests);
    }, exception -> {
        LOG.error(new ParameterizedMessage("fail to get checkpoint [{}]", modelId, exception));
        nodeStateManager.setException(detectorId, exception);
        processCheckpointIteration(index + 1, toProcess, successfulRequests, retryableRequests);
    });
}

Example 12 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class PreviewAnomalyDetectorActionTests method testPreviewResponse.

@Test
public void testPreviewResponse() throws Exception {
    BytesStreamOutput out = new BytesStreamOutput();
    AnomalyDetector detector = TestHelpers.randomAnomalyDetector(ImmutableMap.of("testKey", "testValue"), Instant.now());
    AnomalyResult result = TestHelpers.randomHCADAnomalyDetectResult(0.8d, 0d);
    PreviewAnomalyDetectorResponse response = new PreviewAnomalyDetectorResponse(ImmutableList.of(result), detector);
    response.writeTo(out);
    NamedWriteableAwareStreamInput input = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), writableRegistry());
    PreviewAnomalyDetectorResponse newResponse = new PreviewAnomalyDetectorResponse(input);
    Assert.assertNotNull(newResponse.toXContent(TestHelpers.builder(), ToXContent.EMPTY_PARAMS));
}

Example 13 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class PreviewAnomalyDetectorTransportActionTests method testPreviewTransportActionWithDetector.

@SuppressWarnings("unchecked")
@Test
public void testPreviewTransportActionWithDetector() throws IOException, InterruptedException {
    final CountDownLatch inProgressLatch = new CountDownLatch(1);
    CreateIndexResponse createResponse = TestHelpers.createIndex(client().admin(), AnomalyDetector.ANOMALY_DETECTORS_INDEX, AnomalyDetectionIndices.getAnomalyDetectorMappings());
    Assert.assertNotNull(createResponse);
    AnomalyDetector detector = TestHelpers.randomAnomalyDetector(ImmutableMap.of("testKey", "testValue"), Instant.now());
    IndexRequest indexRequest = new IndexRequest(AnomalyDetector.ANOMALY_DETECTORS_INDEX).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).source(detector.toXContent(XContentFactory.jsonBuilder(), RestHandlerUtils.XCONTENT_WITH_TYPE));
    IndexResponse indexResponse = client().index(indexRequest).actionGet(5_000);
    assertEquals(RestStatus.CREATED, indexResponse.status());
    PreviewAnomalyDetectorRequest request = new PreviewAnomalyDetectorRequest(null, indexResponse.getId(), Instant.now(), Instant.now());
    ActionListener<PreviewAnomalyDetectorResponse> previewResponse = new ActionListener<PreviewAnomalyDetectorResponse>() {

        @Override
        public void onResponse(PreviewAnomalyDetectorResponse response) {
            try {
                XContentBuilder previewBuilder = response.toXContent(TestHelpers.builder(), ToXContent.EMPTY_PARAMS);
                Assert.assertNotNull(previewBuilder);
                Map<String, Object> map = TestHelpers.XContentBuilderToMap(previewBuilder);
                List<AnomalyResult> results = (List<AnomalyResult>) map.get("anomaly_result");
                Assert.assertNotNull(results);
                Assert.assertTrue(results.size() > 0);
                inProgressLatch.countDown();
            } catch (IOException e) {
                // Should not reach here
                Assert.assertTrue(false);
            }
        }

        @Override
        public void onFailure(Exception e) {
            // onFailure should not be called
            Assert.assertTrue(false);
        }
    };
    doReturn(TestHelpers.randomThresholdingResults()).when(modelManager).getPreviewResults(any(), anyInt());
    doAnswer(responseMock -> {
        Long startTime = responseMock.getArgument(1);
        ActionListener<Features> listener = responseMock.getArgument(3);
        listener.onResponse(TestHelpers.randomFeatures());
        return null;
    }).when(featureManager).getPreviewFeatures(anyObject(), anyLong(), anyLong(), any());
    action.doExecute(task, request, previewResponse);
    assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS));
}

Example 14 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class PreviewAnomalyDetectorTransportActionTests method testPreviewTransportAction.

@SuppressWarnings("unchecked")
@Test
public void testPreviewTransportAction() throws IOException, InterruptedException {
    final CountDownLatch inProgressLatch = new CountDownLatch(1);
    AnomalyDetector detector = TestHelpers.randomAnomalyDetector(ImmutableMap.of("testKey", "testValue"), Instant.now());
    PreviewAnomalyDetectorRequest request = new PreviewAnomalyDetectorRequest(detector, detector.getDetectorId(), Instant.now(), Instant.now());
    ActionListener<PreviewAnomalyDetectorResponse> previewResponse = new ActionListener<PreviewAnomalyDetectorResponse>() {

        @Override
        public void onResponse(PreviewAnomalyDetectorResponse response) {
            try {
                XContentBuilder previewBuilder = response.toXContent(TestHelpers.builder(), ToXContent.EMPTY_PARAMS);
                Assert.assertNotNull(previewBuilder);
                Map<String, Object> map = TestHelpers.XContentBuilderToMap(previewBuilder);
                List<AnomalyResult> results = (List<AnomalyResult>) map.get("anomaly_result");
                Assert.assertNotNull(results);
                Assert.assertTrue(results.size() > 0);
                inProgressLatch.countDown();
            } catch (IOException e) {
                // Should not reach here
                Assert.assertTrue(false);
            }
        }

        @Override
        public void onFailure(Exception e) {
            // onFailure should not be called
            Assert.assertTrue(false);
        }
    };
    doReturn(TestHelpers.randomThresholdingResults()).when(modelManager).getPreviewResults(any(), anyInt());
    doAnswer(responseMock -> {
        Long startTime = responseMock.getArgument(1);
        ActionListener<Features> listener = responseMock.getArgument(3);
        listener.onResponse(TestHelpers.randomFeatures());
        return null;
    }).when(featureManager).getPreviewFeatures(anyObject(), anyLong(), anyLong(), any());
    action.doExecute(task, request, previewResponse);
    assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS));
}

Example 15 with AnomalyResult

use of org.opensearch.ad.model.AnomalyResult in project anomaly-detection by opensearch-project.

the class EntityResultTransportAction method onGetDetector.

private ActionListener<Optional<AnomalyDetector>> onGetDetector(ActionListener<AcknowledgedResponse> listener, String detectorId, EntityResultRequest request, Optional<Exception> prevException) {
    return ActionListener.wrap(detectorOptional -> {
        if (!detectorOptional.isPresent()) {
            listener.onFailure(new EndRunException(detectorId, "AnomalyDetector is not available.", true));
            return;
        }
        AnomalyDetector detector = detectorOptional.get();
        if (request.getEntities() == null) {
            listener.onResponse(null);
            return;
        }
        Instant executionStartTime = Instant.now();
        Map<Entity, double[]> cacheMissEntities = new HashMap<>();
        for (Entry<Entity, double[]> entityEntry : request.getEntities().entrySet()) {
            Entity categoricalValues = entityEntry.getKey();
            if (isEntityeFromOldNodeMsg(categoricalValues) && detector.getCategoryField() != null && detector.getCategoryField().size() == 1) {
                Map<String, String> attrValues = categoricalValues.getAttributes();
                // handle a request from a version before OpenSearch 1.1.
                categoricalValues = Entity.createSingleAttributeEntity(detector.getCategoryField().get(0), attrValues.get(CommonName.EMPTY_FIELD));
            }
            Optional<String> modelIdOptional = categoricalValues.getModelId(detectorId);
            if (false == modelIdOptional.isPresent()) {
                continue;
            }
            String modelId = modelIdOptional.get();
            double[] datapoint = entityEntry.getValue();
            ModelState<EntityModel> entityModel = cache.get().get(modelId, detector);
            if (entityModel == null) {
                // cache miss
                cacheMissEntities.put(categoricalValues, datapoint);
                continue;
            }
            ThresholdingResult result = modelManager.getAnomalyResultForEntity(datapoint, entityModel, modelId, categoricalValues, detector.getShingleSize());
            // So many OpenSearchRejectedExecutionException if we write no matter what
            if (result.getRcfScore() > 0) {
                AnomalyResult resultToSave = result.toAnomalyResult(detector, Instant.ofEpochMilli(request.getStart()), Instant.ofEpochMilli(request.getEnd()), executionStartTime, Instant.now(), ParseUtils.getFeatureData(datapoint, detector), categoricalValues, indexUtil.getSchemaVersion(ADIndex.RESULT), modelId, null, null);
                resultWriteQueue.put(new ResultWriteRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, result.getGrade() > 0 ? RequestPriority.HIGH : RequestPriority.MEDIUM, resultToSave, detector.getResultIndex()));
            }
        }
        // split hot and cold entities
        Pair<List<Entity>, List<Entity>> hotColdEntities = cache.get().selectUpdateCandidate(cacheMissEntities.keySet(), detectorId, detector);
        List<EntityFeatureRequest> hotEntityRequests = new ArrayList<>();
        List<EntityFeatureRequest> coldEntityRequests = new ArrayList<>();
        for (Entity hotEntity : hotColdEntities.getLeft()) {
            double[] hotEntityValue = cacheMissEntities.get(hotEntity);
            if (hotEntityValue == null) {
                LOG.error(new ParameterizedMessage("feature value should not be null: [{}]", hotEntity));
                continue;
            }
            hotEntityRequests.add(new EntityFeatureRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, // hot entities has MEDIUM priority
            RequestPriority.MEDIUM, hotEntity, hotEntityValue, request.getStart()));
        }
        for (Entity coldEntity : hotColdEntities.getRight()) {
            double[] coldEntityValue = cacheMissEntities.get(coldEntity);
            if (coldEntityValue == null) {
                LOG.error(new ParameterizedMessage("feature value should not be null: [{}]", coldEntity));
                continue;
            }
            coldEntityRequests.add(new EntityFeatureRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, // cold entities has LOW priority
            RequestPriority.LOW, coldEntity, coldEntityValue, request.getStart()));
        }
        checkpointReadQueue.putAll(hotEntityRequests);
        coldEntityQueue.putAll(coldEntityRequests);
        // respond back
        if (prevException.isPresent()) {
            listener.onFailure(prevException.get());
        } else {
            listener.onResponse(new AcknowledgedResponse(true));
        }
    }, exception -> {
        LOG.error(new ParameterizedMessage("fail to get entity's anomaly grade for detector [{}]: start: [{}], end: [{}]", detectorId, request.getStart(), request.getEnd()), exception);
        listener.onFailure(exception);
    });
}