Search in sources :

Example 21 with Entity

use of org.opensearch.ad.model.Entity in project anomaly-detection by opensearch-project.

the class PriorityCacheTests method testClearMemory.

/*
     * Test the scenario:
     * 1. A detector's buffer uses dedicated and shared memory
     * 2. a new detector's buffer is created and triggers clearMemory (every new
     *  CacheBuffer creation will trigger it)
     * 3. clearMemory found we can reclaim shared memory
     */
public void testClearMemory() {
    for (int i = 0; i < 2; i++) {
        // bypass doorkeeper
        cacheProvider.get(entity2.getModelId(detectorId).get(), detector);
    }
    for (int i = 0; i < 10; i++) {
        // bypass doorkeeper and make entity1 have higher frequency
        cacheProvider.get(entity1.getModelId(detectorId).get(), detector);
    }
    // put modelState5 in dedicated and modelState6 in shared cache
    when(memoryTracker.canAllocate(anyLong())).thenReturn(true);
    cacheProvider.hostIfPossible(detector, modelState1);
    cacheProvider.hostIfPossible(detector, modelState2);
    // two entities get inserted to cache
    assertTrue(null != cacheProvider.get(entity1.getModelId(detectorId).get(), detector));
    assertTrue(null != cacheProvider.get(entity2.getModelId(detectorId).get(), detector));
    Entity entity5 = Entity.createSingleAttributeEntity("attributeName1", "attributeVal5");
    when(memoryTracker.memoryToShed()).thenReturn(memoryPerEntity);
    for (int i = 0; i < 2; i++) {
        // bypass doorkeeper, CacheBuffer created, and trigger clearMemory
        cacheProvider.get(entity5.getModelId(detectorId2).get(), detector2);
    }
    assertTrue(null != cacheProvider.get(entity1.getModelId(detectorId).get(), detector));
    // entity 2 removed
    assertTrue(null == cacheProvider.get(entity2.getModelId(detectorId).get(), detector));
    assertTrue(null == cacheProvider.get(entity5.getModelId(detectorId2).get(), detector));
}
Also used : Entity(org.opensearch.ad.model.Entity)

Example 22 with Entity

use of org.opensearch.ad.model.Entity in project anomaly-detection by opensearch-project.

the class EntityProfileRunnerTests method testModel.

public void testModel() throws InterruptedException {
    setUpExecuteEntityProfileAction(InittedEverResultStatus.INITTED);
    EntityProfile.Builder expectedProfile = new EntityProfile.Builder();
    ModelProfileOnNode modelProfile = new ModelProfileOnNode(nodeId, new ModelProfile(modelId, entity, modelSize));
    expectedProfile.modelProfile(modelProfile);
    final CountDownLatch inProgressLatch = new CountDownLatch(1);
    runner.profile(detectorId, entity, model, ActionListener.wrap(response -> {
        assertEquals(expectedProfile.build(), response);
        inProgressLatch.countDown();
    }, exception -> {
        assertTrue("Should not reach here", false);
        inProgressLatch.countDown();
    }));
    assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) ModelProfile(org.opensearch.ad.model.ModelProfile) DocValueFormat(org.opensearch.search.DocValueFormat) SearchHits(org.opensearch.search.SearchHits) EntityProfileName(org.opensearch.ad.model.EntityProfileName) ModelProfileOnNode(org.opensearch.ad.model.ModelProfileOnNode) HashSet(java.util.HashSet) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) EntityProfileResponse(org.opensearch.ad.transport.EntityProfileResponse) Mockito.doAnswer(org.mockito.Mockito.doAnswer) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) SearchRequest(org.opensearch.action.search.SearchRequest) ActionListener(org.opensearch.action.ActionListener) SearchResponse(org.opensearch.action.search.SearchResponse) GetResponse(org.opensearch.action.get.GetResponse) EntityProfile(org.opensearch.ad.model.EntityProfile) Collections.emptyMap(java.util.Collections.emptyMap) Client(org.opensearch.client.Client) CommonName(org.opensearch.ad.constant.CommonName) InitProgressProfile(org.opensearch.ad.model.InitProgressProfile) SearchHit(org.opensearch.search.SearchHit) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) Set(java.util.Set) IOException(java.io.IOException) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) TimeUnit(java.util.concurrent.TimeUnit) TotalHits(org.apache.lucene.search.TotalHits) CountDownLatch(java.util.concurrent.CountDownLatch) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) Entity(org.opensearch.ad.model.Entity) SearchResponseSections(org.opensearch.action.search.SearchResponseSections) EntityState(org.opensearch.ad.model.EntityState) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) EntityProfileAction(org.opensearch.ad.transport.EntityProfileAction) Text(org.opensearch.common.text.Text) Mockito.mock(org.mockito.Mockito.mock) ModelProfileOnNode(org.opensearch.ad.model.ModelProfileOnNode) EntityProfile(org.opensearch.ad.model.EntityProfile) CountDownLatch(java.util.concurrent.CountDownLatch) ModelProfile(org.opensearch.ad.model.ModelProfile)

Example 23 with Entity

use of org.opensearch.ad.model.Entity in project anomaly-detection by opensearch-project.

the class EntityProfileRunnerTests method stateTestTemplate.

public void stateTestTemplate(InittedEverResultStatus returnedState, EntityState expectedState) throws InterruptedException {
    setUpExecuteEntityProfileAction(returnedState);
    final CountDownLatch inProgressLatch = new CountDownLatch(1);
    runner.profile(detectorId, entity, state, ActionListener.wrap(response -> {
        assertEquals(expectedState, response.getState());
        inProgressLatch.countDown();
    }, exception -> {
        assertTrue("Should not reach here", false);
        inProgressLatch.countDown();
    }));
    assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) ModelProfile(org.opensearch.ad.model.ModelProfile) DocValueFormat(org.opensearch.search.DocValueFormat) SearchHits(org.opensearch.search.SearchHits) EntityProfileName(org.opensearch.ad.model.EntityProfileName) ModelProfileOnNode(org.opensearch.ad.model.ModelProfileOnNode) HashSet(java.util.HashSet) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) EntityProfileResponse(org.opensearch.ad.transport.EntityProfileResponse) Mockito.doAnswer(org.mockito.Mockito.doAnswer) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) SearchRequest(org.opensearch.action.search.SearchRequest) ActionListener(org.opensearch.action.ActionListener) SearchResponse(org.opensearch.action.search.SearchResponse) GetResponse(org.opensearch.action.get.GetResponse) EntityProfile(org.opensearch.ad.model.EntityProfile) Collections.emptyMap(java.util.Collections.emptyMap) Client(org.opensearch.client.Client) CommonName(org.opensearch.ad.constant.CommonName) InitProgressProfile(org.opensearch.ad.model.InitProgressProfile) SearchHit(org.opensearch.search.SearchHit) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) Set(java.util.Set) IOException(java.io.IOException) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) TimeUnit(java.util.concurrent.TimeUnit) TotalHits(org.apache.lucene.search.TotalHits) CountDownLatch(java.util.concurrent.CountDownLatch) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) Entity(org.opensearch.ad.model.Entity) SearchResponseSections(org.opensearch.action.search.SearchResponseSections) EntityState(org.opensearch.ad.model.EntityState) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) EntityProfileAction(org.opensearch.ad.transport.EntityProfileAction) Text(org.opensearch.common.text.Text) Mockito.mock(org.mockito.Mockito.mock) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 24 with Entity

use of org.opensearch.ad.model.Entity in project anomaly-detection by opensearch-project.

the class EntityProfileRunnerTests method testNotMultiEntityDetector.

@SuppressWarnings("unchecked")
public void testNotMultiEntityDetector() throws IOException, InterruptedException {
    detector = TestHelpers.randomAnomalyDetectorWithInterval(new IntervalTimeConfiguration(detectorIntervalMin, ChronoUnit.MINUTES));
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        GetRequest request = (GetRequest) args[0];
        ActionListener<GetResponse> listener = (ActionListener<GetResponse>) args[1];
        String indexName = request.index();
        if (indexName.equals(ANOMALY_DETECTORS_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(detector, detector.getDetectorId(), AnomalyDetector.ANOMALY_DETECTORS_INDEX));
        }
        return null;
    }).when(client).get(any(), any());
    final CountDownLatch inProgressLatch = new CountDownLatch(1);
    runner.profile(detectorId, entity, state, ActionListener.wrap(response -> {
        assertTrue("Should not reach here", false);
        inProgressLatch.countDown();
    }, exception -> {
        assertTrue(exception.getMessage().contains(EntityProfileRunner.NOT_HC_DETECTOR_ERR_MSG));
        inProgressLatch.countDown();
    }));
    assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) ModelProfile(org.opensearch.ad.model.ModelProfile) DocValueFormat(org.opensearch.search.DocValueFormat) SearchHits(org.opensearch.search.SearchHits) EntityProfileName(org.opensearch.ad.model.EntityProfileName) ModelProfileOnNode(org.opensearch.ad.model.ModelProfileOnNode) HashSet(java.util.HashSet) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) EntityProfileResponse(org.opensearch.ad.transport.EntityProfileResponse) Mockito.doAnswer(org.mockito.Mockito.doAnswer) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) SearchRequest(org.opensearch.action.search.SearchRequest) ActionListener(org.opensearch.action.ActionListener) SearchResponse(org.opensearch.action.search.SearchResponse) GetResponse(org.opensearch.action.get.GetResponse) EntityProfile(org.opensearch.ad.model.EntityProfile) Collections.emptyMap(java.util.Collections.emptyMap) Client(org.opensearch.client.Client) CommonName(org.opensearch.ad.constant.CommonName) InitProgressProfile(org.opensearch.ad.model.InitProgressProfile) SearchHit(org.opensearch.search.SearchHit) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) Set(java.util.Set) IOException(java.io.IOException) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) TimeUnit(java.util.concurrent.TimeUnit) TotalHits(org.apache.lucene.search.TotalHits) CountDownLatch(java.util.concurrent.CountDownLatch) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) Entity(org.opensearch.ad.model.Entity) SearchResponseSections(org.opensearch.action.search.SearchResponseSections) EntityState(org.opensearch.ad.model.EntityState) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) EntityProfileAction(org.opensearch.ad.transport.EntityProfileAction) Text(org.opensearch.common.text.Text) Mockito.mock(org.mockito.Mockito.mock) ActionListener(org.opensearch.action.ActionListener) GetRequest(org.opensearch.action.get.GetRequest) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) CountDownLatch(java.util.concurrent.CountDownLatch) GetResponse(org.opensearch.action.get.GetResponse)

Example 25 with Entity

use of org.opensearch.ad.model.Entity in project anomaly-detection by opensearch-project.

the class EntityProfileRunnerTests method testJobIndexNotFound.

@SuppressWarnings("unchecked")
public void testJobIndexNotFound() throws InterruptedException {
    setUpExecuteEntityProfileAction(InittedEverResultStatus.INITTED);
    final CountDownLatch inProgressLatch = new CountDownLatch(1);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        GetRequest request = (GetRequest) args[0];
        ActionListener<GetResponse> listener = (ActionListener<GetResponse>) args[1];
        String indexName = request.index();
        if (indexName.equals(ANOMALY_DETECTORS_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(detector, detector.getDetectorId(), AnomalyDetector.ANOMALY_DETECTORS_INDEX));
        } else if (indexName.equals(ANOMALY_DETECTOR_JOB_INDEX)) {
            listener.onFailure(new IndexNotFoundException(ANOMALY_DETECTOR_JOB_INDEX));
        }
        return null;
    }).when(client).get(any(), any());
    EntityProfile expectedProfile = new EntityProfile.Builder().build();
    runner.profile(detectorId, entity, initNInfo, ActionListener.wrap(response -> {
        assertEquals(expectedProfile, response);
        inProgressLatch.countDown();
    }, exception -> {
        LOG.error("Unexpected error", exception);
        assertTrue("Should not reach here", false);
        inProgressLatch.countDown();
    }));
    assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS));
}
Also used : ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Arrays(java.util.Arrays) ModelProfile(org.opensearch.ad.model.ModelProfile) DocValueFormat(org.opensearch.search.DocValueFormat) SearchHits(org.opensearch.search.SearchHits) EntityProfileName(org.opensearch.ad.model.EntityProfileName) ModelProfileOnNode(org.opensearch.ad.model.ModelProfileOnNode) HashSet(java.util.HashSet) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) ANOMALY_DETECTOR_JOB_INDEX(org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX) EntityProfileResponse(org.opensearch.ad.transport.EntityProfileResponse) Mockito.doAnswer(org.mockito.Mockito.doAnswer) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) SearchRequest(org.opensearch.action.search.SearchRequest) ActionListener(org.opensearch.action.ActionListener) SearchResponse(org.opensearch.action.search.SearchResponse) GetResponse(org.opensearch.action.get.GetResponse) EntityProfile(org.opensearch.ad.model.EntityProfile) Collections.emptyMap(java.util.Collections.emptyMap) Client(org.opensearch.client.Client) CommonName(org.opensearch.ad.constant.CommonName) InitProgressProfile(org.opensearch.ad.model.InitProgressProfile) SearchHit(org.opensearch.search.SearchHit) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ANOMALY_DETECTORS_INDEX(org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX) GetRequest(org.opensearch.action.get.GetRequest) Set(java.util.Set) IOException(java.io.IOException) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) TimeUnit(java.util.concurrent.TimeUnit) TotalHits(org.apache.lucene.search.TotalHits) CountDownLatch(java.util.concurrent.CountDownLatch) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ChronoUnit(java.time.temporal.ChronoUnit) Entity(org.opensearch.ad.model.Entity) SearchResponseSections(org.opensearch.action.search.SearchResponseSections) EntityState(org.opensearch.ad.model.EntityState) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) EntityProfileAction(org.opensearch.ad.transport.EntityProfileAction) Text(org.opensearch.common.text.Text) Mockito.mock(org.mockito.Mockito.mock) ActionListener(org.opensearch.action.ActionListener) GetRequest(org.opensearch.action.get.GetRequest) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) EntityProfile(org.opensearch.ad.model.EntityProfile) CountDownLatch(java.util.concurrent.CountDownLatch) GetResponse(org.opensearch.action.get.GetResponse)

Aggregations

Entity (org.opensearch.ad.model.Entity)66 ActionListener (org.opensearch.action.ActionListener)37 ArrayList (java.util.ArrayList)36 List (java.util.List)35 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)34 Client (org.opensearch.client.Client)31 Optional (java.util.Optional)30 SearchRequest (org.opensearch.action.search.SearchRequest)30 HashMap (java.util.HashMap)29 Map (java.util.Map)28 IOException (java.io.IOException)26 LogManager (org.apache.logging.log4j.LogManager)26 Logger (org.apache.logging.log4j.Logger)26 SearchResponse (org.opensearch.action.search.SearchResponse)25 Settings (org.opensearch.common.settings.Settings)23 Set (java.util.Set)22 GetRequest (org.opensearch.action.get.GetRequest)22 ANOMALY_DETECTORS_INDEX (org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX)22 AnomalyDetectorJob (org.opensearch.ad.model.AnomalyDetectorJob)22 AnomalyDetectionException (org.opensearch.ad.common.exception.AnomalyDetectionException)21