Examples with EntityCache org.opensearch.ad.caching.EntityCache used on opensource projects

Example 1 with EntityCache

use of org.opensearch.ad.caching.EntityCache in project anomaly-detection by opensearch-project.

the class ADStatsTests method setup.

@Before
public void setup() {
    MockitoAnnotations.initMocks(this);
    // sampleSize * numberOfTrees has to be larger than 1. Otherwise, RCF reports errors.
    rcf = RandomCutForest.builder().dimensions(1).sampleSize(2).numberOfTrees(1).build();
    thresholdingModel = new HybridThresholdingModel(1e-8, 1e-5, 200, 10_000, 2, 5_000_000);
    List<ModelState<?>> modelsInformation = new ArrayList<>(Arrays.asList(new ModelState<>(rcf, "rcf-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(rcf, "rcf-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f)));
    when(modelManager.getAllModels()).thenReturn(modelsInformation);
    ModelState<EntityModel> entityModel1 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    ModelState<EntityModel> entityModel2 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    List<ModelState<?>> entityModelsInformation = new ArrayList<>(Arrays.asList(entityModel1, entityModel2));
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getAllModels()).thenReturn(entityModelsInformation);
    IndexUtils indexUtils = mock(IndexUtils.class);
    when(indexUtils.getIndexHealthStatus(anyString())).thenReturn("yellow");
    when(indexUtils.getNumberOfDocumentsInIndex(anyString())).thenReturn(100L);
    clusterStatName1 = "clusterStat1";
    clusterStatName2 = "clusterStat2";
    nodeStatName1 = "nodeStat1";
    nodeStatName2 = "nodeStat2";
    Settings settings = Settings.builder().put(MAX_MODEL_SIZE_PER_NODE.getKey(), 10).build();
    ClusterService clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(MAX_MODEL_SIZE_PER_NODE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    statsMap = new HashMap<String, ADStat<?>>() {

        {
            put(nodeStatName1, new ADStat<>(false, new CounterSupplier()));
            put(nodeStatName2, new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider, settings, clusterService)));
            put(clusterStatName1, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index1")));
            put(clusterStatName2, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index2")));
        }
    };
    adStats = new ADStats(statsMap);
}

Example 2 with EntityCache

use of org.opensearch.ad.caching.EntityCache in project anomaly-detection by opensearch-project.

the class ADStatsNodesTransportActionTests method setUp.

@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    Client client = client();
    Clock clock = mock(Clock.class);
    Throttler throttler = new Throttler(clock);
    ThreadPool threadPool = mock(ThreadPool.class);
    IndexNameExpressionResolver indexNameResolver = mock(IndexNameExpressionResolver.class);
    IndexUtils indexUtils = new IndexUtils(client, new ClientUtil(Settings.EMPTY, client, throttler, threadPool), clusterService(), indexNameResolver);
    ModelManager modelManager = mock(ModelManager.class);
    CacheProvider cacheProvider = mock(CacheProvider.class);
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    clusterStatName1 = "clusterStat1";
    clusterStatName2 = "clusterStat2";
    nodeStatName1 = "nodeStat1";
    nodeStatName2 = "nodeStat2";
    Settings settings = Settings.builder().put(MAX_MODEL_SIZE_PER_NODE.getKey(), 10).build();
    ClusterService clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(MAX_MODEL_SIZE_PER_NODE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    statsMap = new HashMap<String, ADStat<?>>() {

        {
            put(nodeStatName1, new ADStat<>(false, new CounterSupplier()));
            put(nodeStatName2, new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider, settings, clusterService)));
            put(clusterStatName1, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index1")));
            put(clusterStatName2, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index2")));
            put(InternalStatNames.JVM_HEAP_USAGE.getName(), new ADStat<>(true, new SettableSupplier()));
        }
    };
    adStats = new ADStats(statsMap);
    JvmService jvmService = mock(JvmService.class);
    JvmStats jvmStats = mock(JvmStats.class);
    JvmStats.Mem mem = mock(JvmStats.Mem.class);
    when(jvmService.stats()).thenReturn(jvmStats);
    when(jvmStats.getMem()).thenReturn(mem);
    when(mem.getHeapUsedPercent()).thenReturn(randomShort());
    adTaskManager = mock(ADTaskManager.class);
    action = new ADStatsNodesTransportAction(client().threadPool(), clusterService(), mock(TransportService.class), mock(ActionFilters.class), adStats, jvmService, adTaskManager);
}

Example 3 with EntityCache

use of org.opensearch.ad.caching.EntityCache in project anomaly-detection by opensearch-project.

the class ModelsOnNodeSupplierTests method setup.

@Before
public void setup() {
    MockitoAnnotations.initMocks(this);
    clock = Clock.systemUTC();
    rcf = RandomCutForest.builder().dimensions(1).sampleSize(2).numberOfTrees(1).build();
    thresholdingModel = new HybridThresholdingModel(1e-8, 1e-5, 200, 10_000, 2, 5_000_000);
    expectedResults = new ArrayList<>(Arrays.asList(new ModelState<>(rcf, "rcf-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(rcf, "rcf-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f)));
    when(modelManager.getAllModels()).thenReturn(expectedResults);
    ModelState<EntityModel> entityModel1 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    ModelState<EntityModel> entityModel2 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    entityModelsInformation = new ArrayList<>(Arrays.asList(entityModel1, entityModel2));
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getAllModels()).thenReturn(entityModelsInformation);
}

Example 4 with EntityCache

use of org.opensearch.ad.caching.EntityCache in project anomaly-detection by opensearch-project.

the class EntityProfileTransportAction method doExecute.

@Override
protected void doExecute(Task task, EntityProfileRequest request, ActionListener<EntityProfileResponse> listener) {
    String adID = request.getAdID();
    Entity entityValue = request.getEntityValue();
    Optional<String> modelIdOptional = entityValue.getModelId(adID);
    if (false == modelIdOptional.isPresent()) {
        listener.onFailure(new AnomalyDetectionException(adID, NO_MODEL_ID_FOUND_MSG));
        return;
    }
    // we use entity's toString (e.g., app_0) to find its node
    // This should be consistent with how we land a model node in AnomalyResultTransportAction
    Optional<DiscoveryNode> node = hashRing.getOwningNodeWithSameLocalAdVersionForRealtimeAD(entityValue.toString());
    if (false == node.isPresent()) {
        listener.onFailure(new AnomalyDetectionException(adID, NO_NODE_FOUND_MSG));
        return;
    }
    String nodeId = node.get().getId();
    String modelId = modelIdOptional.get();
    DiscoveryNode localNode = clusterService.localNode();
    if (localNode.getId().equals(nodeId)) {
        EntityCache cache = cacheProvider.get();
        Set<EntityProfileName> profilesToCollect = request.getProfilesToCollect();
        EntityProfileResponse.Builder builder = new EntityProfileResponse.Builder();
        if (profilesToCollect.contains(EntityProfileName.ENTITY_INFO)) {
            builder.setActive(cache.isActive(adID, modelId));
            builder.setLastActiveMs(cache.getLastActiveMs(adID, modelId));
        }
        if (profilesToCollect.contains(EntityProfileName.INIT_PROGRESS) || profilesToCollect.contains(EntityProfileName.STATE)) {
            builder.setTotalUpdates(cache.getTotalUpdates(adID, modelId));
        }
        if (profilesToCollect.contains(EntityProfileName.MODELS)) {
            Optional<ModelProfile> modleProfile = cache.getModelProfile(adID, modelId);
            if (modleProfile.isPresent()) {
                builder.setModelProfile(new ModelProfileOnNode(nodeId, modleProfile.get()));
            }
        }
        listener.onResponse(builder.build());
    } else if (request.remoteAddress() == null) {
        // redirect if request comes from local host.
        // If a request comes from remote machine, it is already redirected.
        // One redirection should be enough.
        // We don't want a potential infinite loop due to any bug and thus give up.
        LOG.info("Sending entity profile request to {} for detector {}, entity {}", nodeId, adID, entityValue);
        try {
            transportService.sendRequest(node.get(), EntityProfileAction.NAME, request, option, new TransportResponseHandler<EntityProfileResponse>() {

                @Override
                public EntityProfileResponse read(StreamInput in) throws IOException {
                    return new EntityProfileResponse(in);
                }

                @Override
                public void handleResponse(EntityProfileResponse response) {
                    listener.onResponse(response);
                }

                @Override
                public void handleException(TransportException exp) {
                    listener.onFailure(exp);
                }

                @Override
                public String executor() {
                    return ThreadPool.Names.SAME;
                }
            });
        } catch (Exception e) {
            LOG.error(String.format(Locale.ROOT, "Fail to get entity profile for detector {}, entity {}", adID, entityValue), e);
            listener.onFailure(new AnomalyDetectionException(adID, FAIL_TO_GET_ENTITY_PROFILE_MSG, e));
        }
    } else {
        // Prior to Opensearch 1.1, we map a node using model id in the profile API.
        // This is not consistent how we map node in AnomalyResultTransportAction, where
        // we use entity values. We fixed that bug in Opensearch 1.1. But this can cause
        // issue when a request involving an old node according to model id.
        // The new node finds the entity value does not map to itself, so it redirects to another node.
        // The redirection can cause an infinite loop. This branch breaks the loop and gives up.
        LOG.error("Fail to get entity profile for detector {}, entity {}. Maybe because old and new node" + " are using different keys for the hash ring.", adID, entityValue);
        listener.onFailure(new AnomalyDetectionException(adID, FAIL_TO_GET_ENTITY_PROFILE_MSG));
    }
}

Example 5 with EntityCache

use of org.opensearch.ad.caching.EntityCache in project anomaly-detection by opensearch-project.

the class CheckpointReadWorkerTests method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    clusterService = mock(ClusterService.class);
    clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_MAX_HEAP_PERCENT, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_CONCURRENCY, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_BATCH_SIZE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    state = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    checkpoint = mock(CheckpointDao.class);
    Map.Entry<EntityModel, Instant> entry = new SimpleImmutableEntry<EntityModel, Instant>(state.getModel(), Instant.now());
    when(checkpoint.processGetResponse(any(), anyString())).thenReturn(Optional.of(entry));
    checkpointWriteQueue = mock(CheckpointWriteWorker.class);
    modelManager = mock(ModelManager.class);
    when(modelManager.processEntityCheckpoint(any(), any(), anyString(), anyString(), anyInt())).thenReturn(state);
    when(modelManager.score(any(), anyString(), any())).thenReturn(new ThresholdingResult(0, 1, 0.7));
    coldstartQueue = mock(EntityColdStartWorker.class);
    resultWriteQueue = mock(ResultWriteWorker.class);
    anomalyDetectionIndices = mock(AnomalyDetectionIndices.class);
    cacheProvider = mock(CacheProvider.class);
    entityCache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(entityCache);
    when(entityCache.hostIfPossible(any(), any())).thenReturn(true);
    // Integer.MAX_VALUE makes a huge heap
    worker = new CheckpointReadWorker(Integer.MAX_VALUE, AnomalyDetectorSettings.ENTITY_FEATURE_REQUEST_SIZE_IN_BYTES, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_MAX_HEAP_PERCENT, clusterService, new Random(42), mock(ADCircuitBreakerService.class), threadPool, Settings.EMPTY, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, clock, AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, AnomalyDetectorSettings.QUEUE_MAINTENANCE, modelManager, checkpoint, coldstartQueue, resultWriteQueue, nodeStateManager, anomalyDetectionIndices, cacheProvider, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue);
    request = new EntityFeatureRequest(Integer.MAX_VALUE, detectorId, RequestPriority.MEDIUM, entity, new double[] { 0 }, 0);
    request2 = new EntityFeatureRequest(Integer.MAX_VALUE, detectorId, RequestPriority.MEDIUM, entity2, new double[] { 0 }, 0);
    request3 = new EntityFeatureRequest(Integer.MAX_VALUE, detectorId, RequestPriority.MEDIUM, entity3, new double[] { 0 }, 0);
}