Examples with EntityFeatureRequest org.opensearch.ad.ratelimit.EntityFeatureRequest used on opensource projects

Example 1 with EntityFeatureRequest

use of org.opensearch.ad.ratelimit.EntityFeatureRequest in project anomaly-detection by opensearch-project.

the class EntityResultTransportAction method onGetDetector.

private ActionListener<Optional<AnomalyDetector>> onGetDetector(ActionListener<AcknowledgedResponse> listener, String detectorId, EntityResultRequest request, Optional<Exception> prevException) {
    return ActionListener.wrap(detectorOptional -> {
        if (!detectorOptional.isPresent()) {
            listener.onFailure(new EndRunException(detectorId, "AnomalyDetector is not available.", true));
            return;
        }
        AnomalyDetector detector = detectorOptional.get();
        if (request.getEntities() == null) {
            listener.onResponse(null);
            return;
        }
        Instant executionStartTime = Instant.now();
        Map<Entity, double[]> cacheMissEntities = new HashMap<>();
        for (Entry<Entity, double[]> entityEntry : request.getEntities().entrySet()) {
            Entity categoricalValues = entityEntry.getKey();
            if (isEntityeFromOldNodeMsg(categoricalValues) && detector.getCategoryField() != null && detector.getCategoryField().size() == 1) {
                Map<String, String> attrValues = categoricalValues.getAttributes();
                // handle a request from a version before OpenSearch 1.1.
                categoricalValues = Entity.createSingleAttributeEntity(detector.getCategoryField().get(0), attrValues.get(CommonName.EMPTY_FIELD));
            }
            Optional<String> modelIdOptional = categoricalValues.getModelId(detectorId);
            if (false == modelIdOptional.isPresent()) {
                continue;
            }
            String modelId = modelIdOptional.get();
            double[] datapoint = entityEntry.getValue();
            ModelState<EntityModel> entityModel = cache.get().get(modelId, detector);
            if (entityModel == null) {
                // cache miss
                cacheMissEntities.put(categoricalValues, datapoint);
                continue;
            }
            ThresholdingResult result = modelManager.getAnomalyResultForEntity(datapoint, entityModel, modelId, categoricalValues, detector.getShingleSize());
            // So many OpenSearchRejectedExecutionException if we write no matter what
            if (result.getRcfScore() > 0) {
                AnomalyResult resultToSave = result.toAnomalyResult(detector, Instant.ofEpochMilli(request.getStart()), Instant.ofEpochMilli(request.getEnd()), executionStartTime, Instant.now(), ParseUtils.getFeatureData(datapoint, detector), categoricalValues, indexUtil.getSchemaVersion(ADIndex.RESULT), modelId, null, null);
                resultWriteQueue.put(new ResultWriteRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, result.getGrade() > 0 ? RequestPriority.HIGH : RequestPriority.MEDIUM, resultToSave, detector.getResultIndex()));
            }
        }
        // split hot and cold entities
        Pair<List<Entity>, List<Entity>> hotColdEntities = cache.get().selectUpdateCandidate(cacheMissEntities.keySet(), detectorId, detector);
        List<EntityFeatureRequest> hotEntityRequests = new ArrayList<>();
        List<EntityFeatureRequest> coldEntityRequests = new ArrayList<>();
        for (Entity hotEntity : hotColdEntities.getLeft()) {
            double[] hotEntityValue = cacheMissEntities.get(hotEntity);
            if (hotEntityValue == null) {
                LOG.error(new ParameterizedMessage("feature value should not be null: [{}]", hotEntity));
                continue;
            }
            hotEntityRequests.add(new EntityFeatureRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, // hot entities has MEDIUM priority
            RequestPriority.MEDIUM, hotEntity, hotEntityValue, request.getStart()));
        }
        for (Entity coldEntity : hotColdEntities.getRight()) {
            double[] coldEntityValue = cacheMissEntities.get(coldEntity);
            if (coldEntityValue == null) {
                LOG.error(new ParameterizedMessage("feature value should not be null: [{}]", coldEntity));
                continue;
            }
            coldEntityRequests.add(new EntityFeatureRequest(System.currentTimeMillis() + detector.getDetectorIntervalInMilliseconds(), detectorId, // cold entities has LOW priority
            RequestPriority.LOW, coldEntity, coldEntityValue, request.getStart()));
        }
        checkpointReadQueue.putAll(hotEntityRequests);
        coldEntityQueue.putAll(coldEntityRequests);
        // respond back
        if (prevException.isPresent()) {
            listener.onFailure(prevException.get());
        } else {
            listener.onResponse(new AcknowledgedResponse(true));
        }
    }, exception -> {
        LOG.error(new ParameterizedMessage("fail to get entity's anomaly grade for detector [{}]: start: [{}], end: [{}]", detectorId, request.getStart(), request.getEnd()), exception);
        listener.onFailure(exception);
    });
}

Example 2 with EntityFeatureRequest

use of org.opensearch.ad.ratelimit.EntityFeatureRequest in project anomaly-detection by opensearch-project.

the class MultiEntityResultTests method testCacheSelectionError.

public void testCacheSelectionError() throws IOException, InterruptedException {
    CountDownLatch inProgress = setUpSearchResponse();
    setUpTransportInterceptor(this::entityResultHandler);
    setUpEntityResult(1);
    when(hashRing.getOwningNodeWithSameLocalAdVersionForRealtimeAD(any(String.class))).thenReturn(Optional.of(testNodes[1].discoveryNode()));
    List<Entity> hotEntities = new ArrayList<>();
    Map<String, Object> attrs4 = new HashMap<>();
    attrs4.put(serviceField, app0);
    attrs4.put(hostField, "server_4");
    Entity entity4 = Entity.createEntityByReordering(attrs4);
    hotEntities.add(entity4);
    List<Entity> coldEntities = new ArrayList<>();
    Map<String, Object> attrs5 = new HashMap<>();
    attrs5.put(serviceField, app0);
    attrs5.put(hostField, "server_5");
    Entity entity5 = Entity.createEntityByReordering(attrs5);
    coldEntities.add(entity5);
    when(entityCache.selectUpdateCandidate(any(), any(), any())).thenReturn(Pair.of(hotEntities, coldEntities));
    PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
    action.doExecute(null, request, listener);
    assertTrue(inProgress.await(10000L, TimeUnit.MILLISECONDS));
    // size 0 because cacheMissEntities has no record of these entities
    verify(checkpointReadQueue).putAll(argThat(new ArgumentMatcher<List<EntityFeatureRequest>>() {

        @Override
        public boolean matches(List<EntityFeatureRequest> argument) {
            List<EntityFeatureRequest> arg = (argument);
            LOG.info("size: " + arg.size());
            return arg.size() == 0;
        }
    }));
    verify(coldEntityQueue).putAll(argThat(new ArgumentMatcher<List<EntityFeatureRequest>>() {

        @Override
        public boolean matches(List<EntityFeatureRequest> argument) {
            List<EntityFeatureRequest> arg = (argument);
            LOG.info("size: " + arg.size());
            return arg.size() == 0;
        }
    }));
}

Example 3 with EntityFeatureRequest

use of org.opensearch.ad.ratelimit.EntityFeatureRequest in project anomaly-detection by opensearch-project.

the class MultiEntityResultTests method testCacheSelection.

public void testCacheSelection() throws IOException, InterruptedException {
    CountDownLatch inProgress = setUpSearchResponse();
    setUpTransportInterceptor(this::entityResultHandler);
    when(hashRing.getOwningNodeWithSameLocalAdVersionForRealtimeAD(any(String.class))).thenReturn(Optional.of(testNodes[1].discoveryNode()));
    List<Entity> hotEntities = new ArrayList<>();
    Entity entity1 = Entity.createEntityByReordering(attrs1);
    hotEntities.add(entity1);
    List<Entity> coldEntities = new ArrayList<>();
    Entity entity2 = Entity.createEntityByReordering(attrs2);
    coldEntities.add(entity2);
    provider = mock(CacheProvider.class);
    entityCache = mock(EntityCache.class);
    when(provider.get()).thenReturn(entityCache);
    when(entityCache.selectUpdateCandidate(any(), any(), any())).thenReturn(Pair.of(hotEntities, coldEntities));
    when(entityCache.get(any(), any())).thenReturn(null);
    new EntityResultTransportAction(new ActionFilters(Collections.emptySet()), // since we send requests to testNodes[1]
    testNodes[1].transportService, normalModelManager, adCircuitBreakerService, provider, stateManager, indexUtil, resultWriteQueue, checkpointReadQueue, coldEntityQueue, threadPool);
    PlainActionFuture<AnomalyResultResponse> listener = new PlainActionFuture<>();
    action.doExecute(null, request, listener);
    assertTrue(inProgress.await(10000L, TimeUnit.MILLISECONDS));
    verify(checkpointReadQueue).putAll(argThat(new ArgumentMatcher<List<EntityFeatureRequest>>() {

        @Override
        public boolean matches(List<EntityFeatureRequest> argument) {
            List<EntityFeatureRequest> arg = (argument);
            LOG.info("size: " + arg.size() + " ; element: " + arg.get(0));
            return arg.size() == 1 && arg.get(0).getEntity().equals(entity1);
        }
    }));
    verify(coldEntityQueue).putAll(argThat(new ArgumentMatcher<List<EntityFeatureRequest>>() {

        @Override
        public boolean matches(List<EntityFeatureRequest> argument) {
            List<EntityFeatureRequest> arg = (argument);
            LOG.info("size: " + arg.size() + " ; element: " + arg.get(0));
            return arg.size() == 1 && arg.get(0).getEntity().equals(entity2);
        }
    }));
}