Examples with ModelState org.opensearch.ad.ml.ModelState used on opensource projects

Example 1 with ModelState

use of org.opensearch.ad.ml.ModelState in project anomaly-detection by opensearch-project.

the class PriorityCache method maintenance.

/**
 * Maintain active entity's cache and door keepers.
 *
 * inActiveEntities is a Guava's LRU cache. The data structure itself is
 * gonna evict items if they are inactive for 3 days or its maximum size
 * reached (1 million entries)
 */
@Override
public void maintenance() {
    try {
        // clean up memory if we allocate more memory than we should
        tryClearUpMemory();
        activeEnities.entrySet().stream().forEach(cacheBufferEntry -> {
            String detectorId = cacheBufferEntry.getKey();
            CacheBuffer cacheBuffer = cacheBufferEntry.getValue();
            // remove expired cache buffer
            if (cacheBuffer.expired(modelTtl)) {
                activeEnities.remove(detectorId);
                cacheBuffer.clear();
            } else {
                List<ModelState<EntityModel>> removedStates = cacheBuffer.maintenance();
                for (ModelState<EntityModel> state : removedStates) {
                    addIntoInactiveCache(state);
                }
            }
        });
        maintainInactiveCache();
        doorKeepers.entrySet().stream().forEach(doorKeeperEntry -> {
            String detectorId = doorKeeperEntry.getKey();
            DoorKeeper doorKeeper = doorKeeperEntry.getValue();
            // doorKeeper has its own state ttl
            if (doorKeeper.expired(null)) {
                doorKeepers.remove(detectorId);
            } else {
                doorKeeper.maintenance();
            }
        });
    } catch (Exception e) {
        // will be thrown to ES's transport broadcast handler
        throw new AnomalyDetectionException("Fail to maintain cache", e);
    }
}

Example 2 with ModelState

use of org.opensearch.ad.ml.ModelState in project anomaly-detection by opensearch-project.

the class PriorityCache method selectUpdateCandidate.

@Override
public Pair<List<Entity>, List<Entity>> selectUpdateCandidate(Collection<Entity> cacheMissEntities, String detectorId, AnomalyDetector detector) {
    List<Entity> hotEntities = new ArrayList<>();
    List<Entity> coldEntities = new ArrayList<>();
    CacheBuffer buffer = activeEnities.get(detectorId);
    if (buffer == null) {
        // Since this method is public, need to deal with this case in case of misuse.
        return Pair.of(hotEntities, coldEntities);
    }
    Iterator<Entity> cacheMissEntitiesIter = cacheMissEntities.iterator();
    // current buffer's dedicated cache has free slots
    while (cacheMissEntitiesIter.hasNext() && buffer.dedicatedCacheAvailable()) {
        addEntity(hotEntities, cacheMissEntitiesIter.next(), detectorId);
    }
    while (cacheMissEntitiesIter.hasNext() && memoryTracker.canAllocate(buffer.getMemoryConsumptionPerEntity())) {
        // can allocate in shared cache
        // race conditions can happen when multiple threads evaluating this condition.
        // This is a problem as our AD memory usage is close to full and we put
        // more things than we planned. One model in HCAD is small,
        // it is fine we exceed a little. We have regular maintenance to remove
        // extra memory usage.
        addEntity(hotEntities, cacheMissEntitiesIter.next(), detectorId);
    }
    // check if we can replace anything in dedicated or shared cache
    // have a copy since we need to do the iteration twice: one for
    // dedicated cache and one for shared cache
    List<Entity> otherBufferReplaceCandidates = new ArrayList<>();
    while (cacheMissEntitiesIter.hasNext()) {
        // can replace an entity in the same CacheBuffer living in reserved
        // or shared cache
        // thread safe as each detector has one thread at one time and only the
        // thread can access its buffer.
        Entity entity = cacheMissEntitiesIter.next();
        Optional<String> modelId = entity.getModelId(detectorId);
        if (false == modelId.isPresent()) {
            continue;
        }
        Optional<ModelState<EntityModel>> state = getStateFromInactiveEntiiyCache(modelId.get());
        if (false == state.isPresent()) {
            // not even recorded in inActiveEntities yet because of doorKeeper
            continue;
        }
        ModelState<EntityModel> modelState = state.get();
        float priority = modelState.getPriority();
        if (buffer.canReplaceWithinDetector(priority)) {
            addEntity(hotEntities, entity, detectorId);
        } else {
            // re-evaluate replacement condition in other buffers
            otherBufferReplaceCandidates.add(entity);
        }
    }
    // record current minimum priority among all detectors to save redundant
    // scanning of all CacheBuffers
    CacheBuffer bufferToRemove = null;
    float minPriority = Float.MIN_VALUE;
    // check if we can replace in other CacheBuffer
    cacheMissEntitiesIter = otherBufferReplaceCandidates.iterator();
    while (cacheMissEntitiesIter.hasNext()) {
        // If two threads try to remove the same entity and add their own state, the 2nd remove
        // returns null and only the first one succeeds.
        Entity entity = cacheMissEntitiesIter.next();
        Optional<String> modelId = entity.getModelId(detectorId);
        if (false == modelId.isPresent()) {
            continue;
        }
        Optional<ModelState<EntityModel>> inactiveState = getStateFromInactiveEntiiyCache(modelId.get());
        if (false == inactiveState.isPresent()) {
            // empty state should not stand a chance to replace others
            continue;
        }
        ModelState<EntityModel> state = inactiveState.get();
        float priority = state.getPriority();
        float scaledPriority = buffer.getPriorityTracker().getScaledPriority(priority);
        if (scaledPriority <= minPriority) {
            // not even larger than the minPriority, we can put this to coldEntities
            addEntity(coldEntities, entity, detectorId);
            continue;
        }
        // Float.MIN_VALUE means we need to re-iterate through all CacheBuffers
        if (minPriority == Float.MIN_VALUE) {
            Triple<CacheBuffer, String, Float> bufferToRemoveEntity = canReplaceInSharedCache(buffer, scaledPriority);
            bufferToRemove = bufferToRemoveEntity.getLeft();
            minPriority = bufferToRemoveEntity.getRight();
        }
        if (bufferToRemove != null) {
            addEntity(hotEntities, entity, detectorId);
            // reset minPriority after the replacement so that we need to iterate all CacheBuffer
            // again
            minPriority = Float.MIN_VALUE;
        } else {
            // after trying everything, we can now safely put this to cold entities list
            addEntity(coldEntities, entity, detectorId);
        }
    }
    return Pair.of(hotEntities, coldEntities);
}

Example 3 with ModelState

use of org.opensearch.ad.ml.ModelState in project anomaly-detection by opensearch-project.

the class ADStatsTests method setup.

@Before
public void setup() {
    MockitoAnnotations.initMocks(this);
    // sampleSize * numberOfTrees has to be larger than 1. Otherwise, RCF reports errors.
    rcf = RandomCutForest.builder().dimensions(1).sampleSize(2).numberOfTrees(1).build();
    thresholdingModel = new HybridThresholdingModel(1e-8, 1e-5, 200, 10_000, 2, 5_000_000);
    List<ModelState<?>> modelsInformation = new ArrayList<>(Arrays.asList(new ModelState<>(rcf, "rcf-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(rcf, "rcf-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f)));
    when(modelManager.getAllModels()).thenReturn(modelsInformation);
    ModelState<EntityModel> entityModel1 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    ModelState<EntityModel> entityModel2 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    List<ModelState<?>> entityModelsInformation = new ArrayList<>(Arrays.asList(entityModel1, entityModel2));
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getAllModels()).thenReturn(entityModelsInformation);
    IndexUtils indexUtils = mock(IndexUtils.class);
    when(indexUtils.getIndexHealthStatus(anyString())).thenReturn("yellow");
    when(indexUtils.getNumberOfDocumentsInIndex(anyString())).thenReturn(100L);
    clusterStatName1 = "clusterStat1";
    clusterStatName2 = "clusterStat2";
    nodeStatName1 = "nodeStat1";
    nodeStatName2 = "nodeStat2";
    Settings settings = Settings.builder().put(MAX_MODEL_SIZE_PER_NODE.getKey(), 10).build();
    ClusterService clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(MAX_MODEL_SIZE_PER_NODE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    statsMap = new HashMap<String, ADStat<?>>() {

        {
            put(nodeStatName1, new ADStat<>(false, new CounterSupplier()));
            put(nodeStatName2, new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider, settings, clusterService)));
            put(clusterStatName1, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index1")));
            put(clusterStatName2, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index2")));
        }
    };
    adStats = new ADStats(statsMap);
}

Example 4 with ModelState

use of org.opensearch.ad.ml.ModelState in project anomaly-detection by opensearch-project.

the class ADStatsTests method testADStatsNodeResponseWithEntity.

/**
 * Test we can serialize stats with entity
 * @throws IOException when writeTo and toXContent have errors.
 * @throws JsonPathNotFoundException when json deserialization cannot find a path
 */
@Test
public void testADStatsNodeResponseWithEntity() throws IOException, JsonPathNotFoundException {
    TreeMap<String, String> attributes = new TreeMap<>();
    String name1 = "a";
    String name2 = "b";
    String val1 = "a1";
    String val2 = "a2";
    attributes.put(name1, val1);
    attributes.put(name2, val2);
    String detectorId = "detectorId";
    Entity entity = Entity.createEntityFromOrderedMap(attributes);
    EntityModel entityModel = new EntityModel(entity, null, null);
    Clock clock = mock(Clock.class);
    when(clock.instant()).thenReturn(Instant.now());
    ModelState<EntityModel> state = new ModelState<EntityModel>(entityModel, entity.getModelId(detectorId).get(), detectorId, "entity", clock, 0.1f);
    Map<String, Object> stats = state.getModelStateAsMap();
    // Test serialization
    ADStatsNodeResponse adStatsNodeResponse = new ADStatsNodeResponse(discoveryNode1, stats);
    BytesStreamOutput output = new BytesStreamOutput();
    adStatsNodeResponse.writeTo(output);
    StreamInput streamInput = output.bytes().streamInput();
    ADStatsNodeResponse readResponse = ADStatsNodeResponse.readStats(streamInput);
    assertEquals("readStats failed", readResponse.getStatsMap(), adStatsNodeResponse.getStatsMap());
    // Test toXContent
    XContentBuilder builder = jsonBuilder();
    adStatsNodeResponse.toXContent(builder.startObject(), ToXContent.EMPTY_PARAMS).endObject();
    String json = Strings.toString(builder);
    for (Map.Entry<String, Object> stat : stats.entrySet()) {
        if (stat.getKey().equals(ModelState.LAST_CHECKPOINT_TIME_KEY) || stat.getKey().equals(ModelState.LAST_USED_TIME_KEY)) {
            assertEquals("toXContent does not work", JsonDeserializer.getLongValue(json, stat.getKey()), stat.getValue());
        } else if (stat.getKey().equals(CommonName.ENTITY_KEY)) {
            JsonArray array = JsonDeserializer.getArrayValue(json, stat.getKey());
            assertEquals(2, array.size());
            for (int i = 0; i < 2; i++) {
                JsonElement element = array.get(i);
                String entityName = JsonDeserializer.getChildNode(element, Entity.ATTRIBUTE_NAME_FIELD).getAsString();
                String entityValue = JsonDeserializer.getChildNode(element, Entity.ATTRIBUTE_VALUE_FIELD).getAsString();
                assertTrue(entityName.equals(name1) || entityName.equals(name2));
                if (entityName.equals(name1)) {
                    assertEquals(val1, entityValue);
                } else {
                    assertEquals(val2, entityValue);
                }
            }
        } else {
            assertEquals("toXContent does not work", JsonDeserializer.getTextValue(json, stat.getKey()), stat.getValue());
        }
    }
}

Example 5 with ModelState

use of org.opensearch.ad.ml.ModelState in project anomaly-detection by opensearch-project.

the class PriorityCacheTests method replaceInOtherCacheSetUp.

private void replaceInOtherCacheSetUp() {
    Entity entity5 = Entity.createSingleAttributeEntity("attributeName1", "attributeVal5");
    Entity entity6 = Entity.createSingleAttributeEntity("attributeName1", "attributeVal6");
    ModelState<EntityModel> modelState5 = new ModelState<>(new EntityModel(entity5, new ArrayDeque<>(), null), entity5.getModelId(detectorId2).get(), detectorId2, ModelType.ENTITY.getName(), clock, 0);
    ModelState<EntityModel> modelState6 = new ModelState<>(new EntityModel(entity6, new ArrayDeque<>(), null), entity6.getModelId(detectorId2).get(), detectorId2, ModelType.ENTITY.getName(), clock, 0);
    for (int i = 0; i < 3; i++) {
        // bypass doorkeeper and leave room for lower frequency entity in testSelectToCold
        cacheProvider.get(entity5.getModelId(detectorId2).get(), detector2);
        cacheProvider.get(entity6.getModelId(detectorId2).get(), detector2);
    }
    for (int i = 0; i < 10; i++) {
        // entity1 cannot replace entity2 due to frequency
        cacheProvider.get(entity2.getModelId(detectorId).get(), detector);
    }
    // put modelState5 in dedicated and modelState6 in shared cache
    when(memoryTracker.canAllocate(anyLong())).thenReturn(true);
    cacheProvider.hostIfPossible(detector2, modelState5);
    cacheProvider.hostIfPossible(detector2, modelState6);
    // fill in dedicated cache
    cacheProvider.hostIfPossible(detector, modelState2);
    // don't allow to use shared cache afterwards
    when(memoryTracker.canAllocate(anyLong())).thenReturn(false);
}