Examples with EntityModel org.opensearch.ad.ml.EntityModel used on opensource projects

Example 6 with EntityModel

use of org.opensearch.ad.ml.EntityModel in project anomaly-detection by opensearch-project.

the class MLUtil method createNonEmptyModel.

public static EntityModel createNonEmptyModel(String detectorId, int sampleSize, Entity entity) {
    Queue<double[]> samples = createQueueSamples(sampleSize);
    int numDataPoints = random.nextInt(1000) + AnomalyDetectorSettings.NUM_MIN_SAMPLES;
    ThresholdedRandomCutForest trcf = new ThresholdedRandomCutForest(ThresholdedRandomCutForest.builder().dimensions(1).sampleSize(AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE).numberOfTrees(AnomalyDetectorSettings.NUM_TREES).timeDecay(AnomalyDetectorSettings.TIME_DECAY).outputAfter(AnomalyDetectorSettings.NUM_MIN_SAMPLES).initialAcceptFraction(0.125d).parallelExecutionEnabled(false).internalShinglingEnabled(true).anomalyRate(1 - AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE));
    for (int i = 0; i < numDataPoints; i++) {
        trcf.process(new double[] { random.nextDouble() }, i);
    }
    EntityModel entityModel = new EntityModel(entity, samples, trcf);
    return entityModel;
}

Example 7 with EntityModel

use of org.opensearch.ad.ml.EntityModel in project anomaly-detection by opensearch-project.

the class ADStatsTests method setup.

@Before
public void setup() {
    MockitoAnnotations.initMocks(this);
    // sampleSize * numberOfTrees has to be larger than 1. Otherwise, RCF reports errors.
    rcf = RandomCutForest.builder().dimensions(1).sampleSize(2).numberOfTrees(1).build();
    thresholdingModel = new HybridThresholdingModel(1e-8, 1e-5, 200, 10_000, 2, 5_000_000);
    List<ModelState<?>> modelsInformation = new ArrayList<>(Arrays.asList(new ModelState<>(rcf, "rcf-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(rcf, "rcf-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f)));
    when(modelManager.getAllModels()).thenReturn(modelsInformation);
    ModelState<EntityModel> entityModel1 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    ModelState<EntityModel> entityModel2 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    List<ModelState<?>> entityModelsInformation = new ArrayList<>(Arrays.asList(entityModel1, entityModel2));
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getAllModels()).thenReturn(entityModelsInformation);
    IndexUtils indexUtils = mock(IndexUtils.class);
    when(indexUtils.getIndexHealthStatus(anyString())).thenReturn("yellow");
    when(indexUtils.getNumberOfDocumentsInIndex(anyString())).thenReturn(100L);
    clusterStatName1 = "clusterStat1";
    clusterStatName2 = "clusterStat2";
    nodeStatName1 = "nodeStat1";
    nodeStatName2 = "nodeStat2";
    Settings settings = Settings.builder().put(MAX_MODEL_SIZE_PER_NODE.getKey(), 10).build();
    ClusterService clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(MAX_MODEL_SIZE_PER_NODE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    statsMap = new HashMap<String, ADStat<?>>() {

        {
            put(nodeStatName1, new ADStat<>(false, new CounterSupplier()));
            put(nodeStatName2, new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider, settings, clusterService)));
            put(clusterStatName1, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index1")));
            put(clusterStatName2, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index2")));
        }
    };
    adStats = new ADStats(statsMap);
}

Example 8 with EntityModel

use of org.opensearch.ad.ml.EntityModel in project anomaly-detection by opensearch-project.

the class ADStatsTests method testADStatsNodeResponseWithEntity.

/**
 * Test we can serialize stats with entity
 * @throws IOException when writeTo and toXContent have errors.
 * @throws JsonPathNotFoundException when json deserialization cannot find a path
 */
@Test
public void testADStatsNodeResponseWithEntity() throws IOException, JsonPathNotFoundException {
    TreeMap<String, String> attributes = new TreeMap<>();
    String name1 = "a";
    String name2 = "b";
    String val1 = "a1";
    String val2 = "a2";
    attributes.put(name1, val1);
    attributes.put(name2, val2);
    String detectorId = "detectorId";
    Entity entity = Entity.createEntityFromOrderedMap(attributes);
    EntityModel entityModel = new EntityModel(entity, null, null);
    Clock clock = mock(Clock.class);
    when(clock.instant()).thenReturn(Instant.now());
    ModelState<EntityModel> state = new ModelState<EntityModel>(entityModel, entity.getModelId(detectorId).get(), detectorId, "entity", clock, 0.1f);
    Map<String, Object> stats = state.getModelStateAsMap();
    // Test serialization
    ADStatsNodeResponse adStatsNodeResponse = new ADStatsNodeResponse(discoveryNode1, stats);
    BytesStreamOutput output = new BytesStreamOutput();
    adStatsNodeResponse.writeTo(output);
    StreamInput streamInput = output.bytes().streamInput();
    ADStatsNodeResponse readResponse = ADStatsNodeResponse.readStats(streamInput);
    assertEquals("readStats failed", readResponse.getStatsMap(), adStatsNodeResponse.getStatsMap());
    // Test toXContent
    XContentBuilder builder = jsonBuilder();
    adStatsNodeResponse.toXContent(builder.startObject(), ToXContent.EMPTY_PARAMS).endObject();
    String json = Strings.toString(builder);
    for (Map.Entry<String, Object> stat : stats.entrySet()) {
        if (stat.getKey().equals(ModelState.LAST_CHECKPOINT_TIME_KEY) || stat.getKey().equals(ModelState.LAST_USED_TIME_KEY)) {
            assertEquals("toXContent does not work", JsonDeserializer.getLongValue(json, stat.getKey()), stat.getValue());
        } else if (stat.getKey().equals(CommonName.ENTITY_KEY)) {
            JsonArray array = JsonDeserializer.getArrayValue(json, stat.getKey());
            assertEquals(2, array.size());
            for (int i = 0; i < 2; i++) {
                JsonElement element = array.get(i);
                String entityName = JsonDeserializer.getChildNode(element, Entity.ATTRIBUTE_NAME_FIELD).getAsString();
                String entityValue = JsonDeserializer.getChildNode(element, Entity.ATTRIBUTE_VALUE_FIELD).getAsString();
                assertTrue(entityName.equals(name1) || entityName.equals(name2));
                if (entityName.equals(name1)) {
                    assertEquals(val1, entityValue);
                } else {
                    assertEquals(val2, entityValue);
                }
            }
        } else {
            assertEquals("toXContent does not work", JsonDeserializer.getTextValue(json, stat.getKey()), stat.getValue());
        }
    }
}

Example 9 with EntityModel

use of org.opensearch.ad.ml.EntityModel in project anomaly-detection by opensearch-project.

the class ModelsOnNodeSupplierTests method setup.

@Before
public void setup() {
    MockitoAnnotations.initMocks(this);
    clock = Clock.systemUTC();
    rcf = RandomCutForest.builder().dimensions(1).sampleSize(2).numberOfTrees(1).build();
    thresholdingModel = new HybridThresholdingModel(1e-8, 1e-5, 200, 10_000, 2, 5_000_000);
    expectedResults = new ArrayList<>(Arrays.asList(new ModelState<>(rcf, "rcf-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(rcf, "rcf-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f)));
    when(modelManager.getAllModels()).thenReturn(expectedResults);
    ModelState<EntityModel> entityModel1 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    ModelState<EntityModel> entityModel2 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    entityModelsInformation = new ArrayList<>(Arrays.asList(entityModel1, entityModel2));
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getAllModels()).thenReturn(entityModelsInformation);
}

Example 10 with EntityModel

use of org.opensearch.ad.ml.EntityModel in project anomaly-detection by opensearch-project.

the class PriorityCacheTests method replaceInOtherCacheSetUp.

private void replaceInOtherCacheSetUp() {
    Entity entity5 = Entity.createSingleAttributeEntity("attributeName1", "attributeVal5");
    Entity entity6 = Entity.createSingleAttributeEntity("attributeName1", "attributeVal6");
    ModelState<EntityModel> modelState5 = new ModelState<>(new EntityModel(entity5, new ArrayDeque<>(), null), entity5.getModelId(detectorId2).get(), detectorId2, ModelType.ENTITY.getName(), clock, 0);
    ModelState<EntityModel> modelState6 = new ModelState<>(new EntityModel(entity6, new ArrayDeque<>(), null), entity6.getModelId(detectorId2).get(), detectorId2, ModelType.ENTITY.getName(), clock, 0);
    for (int i = 0; i < 3; i++) {
        // bypass doorkeeper and leave room for lower frequency entity in testSelectToCold
        cacheProvider.get(entity5.getModelId(detectorId2).get(), detector2);
        cacheProvider.get(entity6.getModelId(detectorId2).get(), detector2);
    }
    for (int i = 0; i < 10; i++) {
        // entity1 cannot replace entity2 due to frequency
        cacheProvider.get(entity2.getModelId(detectorId).get(), detector);
    }
    // put modelState5 in dedicated and modelState6 in shared cache
    when(memoryTracker.canAllocate(anyLong())).thenReturn(true);
    cacheProvider.hostIfPossible(detector2, modelState5);
    cacheProvider.hostIfPossible(detector2, modelState6);
    // fill in dedicated cache
    cacheProvider.hostIfPossible(detector, modelState2);
    // don't allow to use shared cache afterwards
    when(memoryTracker.canAllocate(anyLong())).thenReturn(false);
}