Search in sources :

Example 1 with ThresholdingResult

use of org.opensearch.ad.ml.ThresholdingResult in project anomaly-detection by opensearch-project.

the class CheckpointReadWorkerTests method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    clusterService = mock(ClusterService.class);
    clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_MAX_HEAP_PERCENT, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_CONCURRENCY, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_BATCH_SIZE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    state = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    checkpoint = mock(CheckpointDao.class);
    Map.Entry<EntityModel, Instant> entry = new SimpleImmutableEntry<EntityModel, Instant>(state.getModel(), Instant.now());
    when(checkpoint.processGetResponse(any(), anyString())).thenReturn(Optional.of(entry));
    checkpointWriteQueue = mock(CheckpointWriteWorker.class);
    modelManager = mock(ModelManager.class);
    when(modelManager.processEntityCheckpoint(any(), any(), anyString(), anyString(), anyInt())).thenReturn(state);
    when(modelManager.score(any(), anyString(), any())).thenReturn(new ThresholdingResult(0, 1, 0.7));
    coldstartQueue = mock(EntityColdStartWorker.class);
    resultWriteQueue = mock(ResultWriteWorker.class);
    anomalyDetectionIndices = mock(AnomalyDetectionIndices.class);
    cacheProvider = mock(CacheProvider.class);
    entityCache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(entityCache);
    when(entityCache.hostIfPossible(any(), any())).thenReturn(true);
    // Integer.MAX_VALUE makes a huge heap
    worker = new CheckpointReadWorker(Integer.MAX_VALUE, AnomalyDetectorSettings.ENTITY_FEATURE_REQUEST_SIZE_IN_BYTES, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_MAX_HEAP_PERCENT, clusterService, new Random(42), mock(ADCircuitBreakerService.class), threadPool, Settings.EMPTY, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, clock, AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, AnomalyDetectorSettings.QUEUE_MAINTENANCE, modelManager, checkpoint, coldstartQueue, resultWriteQueue, nodeStateManager, anomalyDetectionIndices, cacheProvider, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue);
    request = new EntityFeatureRequest(Integer.MAX_VALUE, detectorId, RequestPriority.MEDIUM, entity, new double[] { 0 }, 0);
    request2 = new EntityFeatureRequest(Integer.MAX_VALUE, detectorId, RequestPriority.MEDIUM, entity2, new double[] { 0 }, 0);
    request3 = new EntityFeatureRequest(Integer.MAX_VALUE, detectorId, RequestPriority.MEDIUM, entity3, new double[] { 0 }, 0);
}
Also used : ClusterSettings(org.opensearch.common.settings.ClusterSettings) EntityCache(org.opensearch.ad.caching.EntityCache) Instant(java.time.Instant) EntityModel(org.opensearch.ad.ml.EntityModel) ModelManager(org.opensearch.ad.ml.ModelManager) CacheProvider(org.opensearch.ad.caching.CacheProvider) ThresholdingResult(org.opensearch.ad.ml.ThresholdingResult) ClusterService(org.opensearch.cluster.service.ClusterService) CheckpointDao(org.opensearch.ad.ml.CheckpointDao) Random(java.util.Random) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) Map(java.util.Map)

Example 2 with ThresholdingResult

use of org.opensearch.ad.ml.ThresholdingResult in project anomaly-detection by opensearch-project.

the class CheckpointReadWorker method onGetDetector.

private ActionListener<Optional<AnomalyDetector>> onGetDetector(EntityFeatureRequest origRequest, int index, String detectorId, List<EntityFeatureRequest> toProcess, Map<String, MultiGetItemResponse> successfulRequests, Set<String> retryableRequests, Optional<Entry<EntityModel, Instant>> checkpoint, Entity entity, String modelId) {
    return ActionListener.wrap(detectorOptional -> {
        if (false == detectorOptional.isPresent()) {
            LOG.warn(new ParameterizedMessage("AnomalyDetector [{}] is not available.", detectorId));
            processCheckpointIteration(index + 1, toProcess, successfulRequests, retryableRequests);
            return;
        }
        AnomalyDetector detector = detectorOptional.get();
        ModelState<EntityModel> modelState = modelManager.processEntityCheckpoint(checkpoint, entity, modelId, detectorId, detector.getShingleSize());
        EntityModel entityModel = modelState.getModel();
        ThresholdingResult result = null;
        if (entityModel.getTrcf().isPresent()) {
            result = modelManager.score(origRequest.getCurrentFeature(), modelId, modelState);
        } else {
            entityModel.addSample(origRequest.getCurrentFeature());
        }
        if (result != null && result.getRcfScore() > 0) {
            AnomalyResult resultToSave = result.toAnomalyResult(detector, Instant.ofEpochMilli(origRequest.getDataStartTimeMillis()), Instant.ofEpochMilli(origRequest.getDataStartTimeMillis() + detector.getDetectorIntervalInMilliseconds()), Instant.now(), Instant.now(), ParseUtils.getFeatureData(origRequest.getCurrentFeature(), detector), entity, indexUtil.getSchemaVersion(ADIndex.RESULT), modelId, null, null);
            resultWriteQueue.put(new ResultWriteRequest(origRequest.getExpirationEpochMs(), detectorId, result.getGrade() > 0 ? RequestPriority.HIGH : RequestPriority.MEDIUM, resultToSave, detector.getResultIndex()));
        }
        // try to load to cache
        boolean loaded = cacheProvider.get().hostIfPossible(detector, modelState);
        if (false == loaded) {
            // not in memory. Maybe cold entities or some other entities
            // have filled the slot while waiting for loading checkpoints.
            checkpointWriteQueue.write(modelState, true, RequestPriority.LOW);
        }
        processCheckpointIteration(index + 1, toProcess, successfulRequests, retryableRequests);
    }, exception -> {
        LOG.error(new ParameterizedMessage("fail to get checkpoint [{}]", modelId, exception));
        nodeStateManager.setException(detectorId, exception);
        processCheckpointIteration(index + 1, toProcess, successfulRequests, retryableRequests);
    });
}
Also used : EntityModel(org.opensearch.ad.ml.EntityModel) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AnomalyResult(org.opensearch.ad.model.AnomalyResult) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ThresholdingResult(org.opensearch.ad.ml.ThresholdingResult)

Example 3 with ThresholdingResult

use of org.opensearch.ad.ml.ThresholdingResult in project anomaly-detection by opensearch-project.

the class MultiEntityResultTests method setUpEntityResult.

private void setUpEntityResult(int nodeIndex, NodeStateManager nodeStateManager) {
    // register entity result action
    new EntityResultTransportAction(new ActionFilters(Collections.emptySet()), // since we send requests to testNodes[1]
    testNodes[nodeIndex].transportService, normalModelManager, adCircuitBreakerService, provider, nodeStateManager, indexUtil, resultWriteQueue, checkpointReadQueue, coldEntityQueue, threadPool);
    when(normalModelManager.getAnomalyResultForEntity(any(), any(), any(), any(), anyInt())).thenReturn(new ThresholdingResult(0, 1, 1));
}
Also used : ActionFilters(org.opensearch.action.support.ActionFilters) ThresholdingResult(org.opensearch.ad.ml.ThresholdingResult)

Example 4 with ThresholdingResult

use of org.opensearch.ad.ml.ThresholdingResult in project anomaly-detection by opensearch-project.

the class RCFResultTests method testCircuitBreaker.

@SuppressWarnings("unchecked")
public void testCircuitBreaker() {
    TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
    ModelManager manager = mock(ModelManager.class);
    ADCircuitBreakerService breakerService = mock(ADCircuitBreakerService.class);
    RCFResultTransportAction action = new RCFResultTransportAction(mock(ActionFilters.class), transportService, manager, breakerService, hashRing);
    doAnswer(invocation -> {
        ActionListener<ThresholdingResult> listener = invocation.getArgument(3);
        listener.onResponse(new ThresholdingResult(grade, 0d, 0.5, totalUpdates, 0, attribution, pastValues, expectedValuesList, likelihood, threshold, 30));
        return null;
    }).when(manager).getTRcfResult(any(String.class), any(String.class), any(double[].class), any(ActionListener.class));
    when(breakerService.isOpen()).thenReturn(true);
    final PlainActionFuture<RCFResultResponse> future = new PlainActionFuture<>();
    RCFResultRequest request = new RCFResultRequest("123", "123-rcf-1", new double[] { 0 });
    action.doExecute(mock(Task.class), request, future);
    expectThrows(LimitExceededException.class, () -> future.actionGet());
}
Also used : Task(org.opensearch.tasks.Task) ADCircuitBreakerService(org.opensearch.ad.breaker.ADCircuitBreakerService) ActionFilters(org.opensearch.action.support.ActionFilters) ModelManager(org.opensearch.ad.ml.ModelManager) ThresholdingResult(org.opensearch.ad.ml.ThresholdingResult) ActionListener(org.opensearch.action.ActionListener) TransportService(org.opensearch.transport.TransportService) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) Transport(org.opensearch.transport.Transport)

Example 5 with ThresholdingResult

use of org.opensearch.ad.ml.ThresholdingResult in project anomaly-detection by opensearch-project.

the class EntityResultTransportAction method onGetDetector.

private ActionListener<Optional<AnomalyDetector>> onGetDetector(ActionListener<AcknowledgedResponse> listener, String detectorId, EntityResultRequest request, Optional<Exception> prevException) {
    return ActionListener.wrap(detectorOptional -> {
        if (!detectorOptional.isPresent()) {
            listener.onFailure(new EndRunException(detectorId, "AnomalyDetector is not available.", true));
            return;
        }
        AnomalyDetector detector = detectorOptional.get();
        if (request.getEntities() == null) {
            listener.onResponse(null);
            return;
        }
        Instant executionStartTime = Instant.now();
        Map<Entity, double[]> cacheMissEntities = new HashMap<>();
        for (Entry<Entity, double[]> entityEntry : request.getEntities().entrySet()) {
            Entity categoricalValues = entityEntry.getKey();
            if (isEntityeFromOldNodeMsg(categoricalValues) && detector.getCategoryField() != null && detector.getCategoryField().size() == 1) {
                Map<String, String> attrValues = categoricalValues.getAttributes();
                // handle a request from a version before OpenSearch 1.1.
                categoricalValues = Entity.createSingleAttributeEntity(detector.getCategoryField().get(0), attrValues.get(CommonName.EMPTY_FIELD));
            }
            Optional<String> modelIdOptional = categoricalValues.getModelId(detectorId);
            if (false == modelIdOptional.isPresent()) {
                continue;
            }
            String modelId = modelIdOptional.get();
            double[] datapoint = entityEntry.getValue();
            ModelState<EntityModel> entityModel = cache.get().get(modelId, detector);
            if (entityModel == null) {
                // cache miss
                cacheMissEntities.put(categoricalValues, datapoint);
                continue;
            }
            ThresholdingResult result = modelManager.getAnomalyResultForEntity(datapoint, entityModel, modelId, categoricalValues, detector.getShingleSize());
            // So many OpenSearchRejectedExecutionException if we write no matter what
            if (result.getRcfScore() > 0) {
                AnomalyResult resultToSave = result.toAnomalyResult(detector, Instant.ofEpochMilli(request.getStart()), Instant.ofEpochMilli(request.getEnd()), executionStartTime, Instant.now(), ParseUtils.getFeatureData(datapoint, detector), categoricalValues, indexUtil.getSchemaVersion(ADIndex.RESULT), modelId, null, null);
                resultWriteQueue.put(new ResultWriteRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, result.getGrade() > 0 ? RequestPriority.HIGH : RequestPriority.MEDIUM, resultToSave, detector.getResultIndex()));
            }
        }
        // split hot and cold entities
        Pair<List<Entity>, List<Entity>> hotColdEntities = cache.get().selectUpdateCandidate(cacheMissEntities.keySet(), detectorId, detector);
        List<EntityFeatureRequest> hotEntityRequests = new ArrayList<>();
        List<EntityFeatureRequest> coldEntityRequests = new ArrayList<>();
        for (Entity hotEntity : hotColdEntities.getLeft()) {
            double[] hotEntityValue = cacheMissEntities.get(hotEntity);
            if (hotEntityValue == null) {
                LOG.error(new ParameterizedMessage("feature value should not be null: [{}]", hotEntity));
                continue;
            }
            hotEntityRequests.add(new EntityFeatureRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, // hot entities has MEDIUM priority
            RequestPriority.MEDIUM, hotEntity, hotEntityValue, request.getStart()));
        }
        for (Entity coldEntity : hotColdEntities.getRight()) {
            double[] coldEntityValue = cacheMissEntities.get(coldEntity);
            if (coldEntityValue == null) {
                LOG.error(new ParameterizedMessage("feature value should not be null: [{}]", coldEntity));
                continue;
            }
            coldEntityRequests.add(new EntityFeatureRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, // cold entities has LOW priority
            RequestPriority.LOW, coldEntity, coldEntityValue, request.getStart()));
        }
        checkpointReadQueue.putAll(hotEntityRequests);
        coldEntityQueue.putAll(coldEntityRequests);
        // respond back
        if (prevException.isPresent()) {
            listener.onFailure(prevException.get());
        } else {
            listener.onResponse(new AcknowledgedResponse(true));
        }
    }, exception -> {
        LOG.error(new ParameterizedMessage("fail to get entity's anomaly grade for detector [{}]: start: [{}], end: [{}]", detectorId, request.getStart(), request.getEnd()), exception);
        listener.onFailure(exception);
    });
}
Also used : Entity(org.opensearch.ad.model.Entity) EndRunException(org.opensearch.ad.common.exception.EndRunException) EntityFeatureRequest(org.opensearch.ad.ratelimit.EntityFeatureRequest) HashMap(java.util.HashMap) Instant(java.time.Instant) EntityModel(org.opensearch.ad.ml.EntityModel) ArrayList(java.util.ArrayList) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ThresholdingResult(org.opensearch.ad.ml.ThresholdingResult) AnomalyResult(org.opensearch.ad.model.AnomalyResult) ArrayList(java.util.ArrayList) List(java.util.List) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) ResultWriteRequest(org.opensearch.ad.ratelimit.ResultWriteRequest)

Aggregations

ThresholdingResult (org.opensearch.ad.ml.ThresholdingResult)12 ModelManager (org.opensearch.ad.ml.ModelManager)7 ActionListener (org.opensearch.action.ActionListener)6 ArrayList (java.util.ArrayList)5 ActionFilters (org.opensearch.action.support.ActionFilters)5 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)5 PlainActionFuture (org.opensearch.action.support.PlainActionFuture)4 AnomalyResult (org.opensearch.ad.model.AnomalyResult)4 TransportService (org.opensearch.transport.TransportService)4 Instant (java.time.Instant)3 Map (java.util.Map)3 ADCircuitBreakerService (org.opensearch.ad.breaker.ADCircuitBreakerService)3 Task (org.opensearch.tasks.Task)3 Transport (org.opensearch.transport.Transport)3 HashMap (java.util.HashMap)2 List (java.util.List)2 Optional (java.util.Optional)2 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)2 Matchers.containsString (org.hamcrest.Matchers.containsString)2 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)2