Search in sources :

Example 16 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class NoPowermockSearchFeatureDaoTests method testGetHighestCountEntitiesExhaustedPages.

@SuppressWarnings("unchecked")
public void testGetHighestCountEntitiesExhaustedPages() throws InterruptedException {
    SearchResponse response1 = createPageResponse(attrs1);
    CompositeAggregation emptyComposite = mock(CompositeAggregation.class);
    when(emptyComposite.getName()).thenReturn(SearchFeatureDao.AGG_NAME_TOP);
    when(emptyComposite.afterKey()).thenReturn(null);
    // empty bucket
    when(emptyComposite.getBuckets()).thenAnswer((Answer<List<CompositeAggregation.Bucket>>) invocation -> {
        return new ArrayList<CompositeAggregation.Bucket>();
    });
    Aggregations emptyAggs = new Aggregations(Collections.singletonList(emptyComposite));
    SearchResponseSections emptySections = new SearchResponseSections(SearchHits.empty(), emptyAggs, null, false, null, null, 1);
    SearchResponse emptyResponse = new SearchResponse(emptySections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY);
    CountDownLatch inProgress = new CountDownLatch(2);
    doAnswer(invocation -> {
        ActionListener<SearchResponse> listener = invocation.getArgument(1);
        inProgress.countDown();
        if (inProgress.getCount() == 1) {
            listener.onResponse(response1);
        } else {
            listener.onResponse(emptyResponse);
        }
        return null;
    }).when(client).search(any(), any());
    ActionListener<List<Entity>> listener = mock(ActionListener.class);
    searchFeatureDao = new SearchFeatureDao(client, xContentRegistry(), interpolator, clientUtil, settings, clusterService, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, clock, 2, 1, 60_000L);
    searchFeatureDao.getHighestCountEntities(detector, 10L, 20L, listener);
    ArgumentCaptor<List<Entity>> captor = ArgumentCaptor.forClass(List.class);
    verify(listener).onResponse(captor.capture());
    List<Entity> result = captor.getValue();
    assertEquals(1, result.size());
    assertEquals(Entity.createEntityByReordering(attrs1), result.get(0));
    // both counts are used in client.search
    assertTrue(inProgress.await(10000L, TimeUnit.MILLISECONDS));
}
Also used : Arrays(java.util.Arrays) IsInstanceOf.instanceOf(org.hamcrest.core.IsInstanceOf.instanceOf) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) AbstractADTest(org.opensearch.ad.AbstractADTest) Releasables(org.opensearch.common.lease.Releasables) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) MockBigArrays(org.opensearch.common.util.MockBigArrays) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) SingleFeatureLinearUniformInterpolator(org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) ZoneOffset(java.time.ZoneOffset) ActionListener(org.opensearch.action.ActionListener) MockPageCacheRecycler(org.opensearch.common.util.MockPageCacheRecycler) DateFormatter(org.opensearch.common.time.DateFormatter) Client(org.opensearch.client.Client) HyperLogLogPlusPlus(org.opensearch.search.aggregations.metrics.HyperLogLogPlusPlus) Clusters(org.opensearch.action.search.SearchResponse.Clusters) BytesRef(org.apache.lucene.util.BytesRef) SearchHit(org.opensearch.search.SearchHit) Collection(java.util.Collection) Feature(org.opensearch.ad.model.Feature) Settings(org.opensearch.common.settings.Settings) StandardCharsets(java.nio.charset.StandardCharsets) InvocationTargetException(java.lang.reflect.InvocationTargetException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) LinearUniformInterpolator(org.opensearch.ad.dataprocessor.LinearUniformInterpolator) Logger(org.apache.logging.log4j.Logger) TestHelpers(org.opensearch.ad.TestHelpers) Entry(java.util.Map.Entry) Optional(java.util.Optional) Mockito.mock(org.mockito.Mockito.mock) InternalDateRange(org.opensearch.search.aggregations.bucket.range.InternalDateRange) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) BucketOrder(org.opensearch.search.aggregations.BucketOrder) InternalFilter(org.opensearch.search.aggregations.bucket.filter.InternalFilter) StringTerms(org.opensearch.search.aggregations.bucket.terms.StringTerms) DocValueFormat(org.opensearch.search.DocValueFormat) AggregatorFactories(org.opensearch.search.aggregations.AggregatorFactories) HashMap(java.util.HashMap) Aggregations(org.opensearch.search.aggregations.Aggregations) AbstractHyperLogLogPlusPlus(org.opensearch.search.aggregations.metrics.AbstractHyperLogLogPlusPlus) SearchHits(org.opensearch.search.SearchHits) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) BitMixer(com.carrotsearch.hppc.BitMixer) Constructor(java.lang.reflect.Constructor) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) InternalOrder(org.opensearch.search.aggregations.InternalOrder) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArgumentCaptor(org.mockito.ArgumentCaptor) ImmutableList(com.google.common.collect.ImmutableList) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) SearchRequest(org.opensearch.action.search.SearchRequest) SearchResponse(org.opensearch.action.search.SearchResponse) ClusterSettings(org.opensearch.common.settings.ClusterSettings) InternalCardinality(org.opensearch.search.aggregations.metrics.InternalCardinality) QueryBuilders(org.opensearch.index.query.QueryBuilders) AbstractHyperLogLog(org.opensearch.search.aggregations.metrics.AbstractHyperLogLog) ClientUtil(org.opensearch.ad.util.ClientUtil) IOException(java.io.IOException) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) CompositeAggregation(org.opensearch.search.aggregations.bucket.composite.CompositeAggregation) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) TotalHits(org.apache.lucene.search.TotalHits) InternalFilters(org.opensearch.search.aggregations.bucket.filter.InternalFilters) ChronoUnit(java.time.temporal.ChronoUnit) Entity(org.opensearch.ad.model.Entity) SearchResponseSections(org.opensearch.action.search.SearchResponseSections) DateFieldMapper(org.opensearch.index.mapper.DateFieldMapper) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) InternalBucket(org.opensearch.search.aggregations.bucket.filter.InternalFilters.InternalBucket) ClusterService(org.opensearch.cluster.service.ClusterService) Clock(java.time.Clock) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) Collections(java.util.Collections) LogManager(org.apache.logging.log4j.LogManager) SumAggregationBuilder(org.opensearch.search.aggregations.metrics.SumAggregationBuilder) Entity(org.opensearch.ad.model.Entity) CompositeAggregation(org.opensearch.search.aggregations.bucket.composite.CompositeAggregation) SearchResponseSections(org.opensearch.action.search.SearchResponseSections) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Aggregations(org.opensearch.search.aggregations.Aggregations) CountDownLatch(java.util.concurrent.CountDownLatch) SearchResponse(org.opensearch.action.search.SearchResponse) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) InternalBucket(org.opensearch.search.aggregations.bucket.filter.InternalFilters.InternalBucket) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList)

Example 17 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class RolloverTests method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    Client client = mock(Client.class);
    indicesClient = mock(IndicesAdminClient.class);
    AdminClient adminClient = mock(AdminClient.class);
    clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS_PER_SHARD, AnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD, AnomalyDetectorSettings.AD_RESULT_HISTORY_RETENTION_PERIOD, AnomalyDetectorSettings.MAX_PRIMARY_SHARDS))));
    clusterName = new ClusterName("test");
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    ThreadPool threadPool = mock(ThreadPool.class);
    Settings settings = Settings.EMPTY;
    when(client.admin()).thenReturn(adminClient);
    when(adminClient.indices()).thenReturn(indicesClient);
    DiscoveryNodeFilterer nodeFilter = mock(DiscoveryNodeFilterer.class);
    numberOfNodes = 2;
    when(nodeFilter.getNumberOfEligibleDataNodes()).thenReturn(numberOfNodes);
    adIndices = new AnomalyDetectionIndices(client, clusterService, threadPool, settings, nodeFilter, AnomalyDetectorSettings.MAX_UPDATE_RETRY_TIMES);
    clusterAdminClient = mock(ClusterAdminClient.class);
    when(adminClient.cluster()).thenReturn(clusterAdminClient);
    doAnswer(invocation -> {
        ClusterStateRequest clusterStateRequest = invocation.getArgument(0);
        assertEquals(AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN, clusterStateRequest.indices()[0]);
        @SuppressWarnings("unchecked") ActionListener<ClusterStateResponse> listener = (ActionListener<ClusterStateResponse>) invocation.getArgument(1);
        listener.onResponse(new ClusterStateResponse(clusterName, clusterState, true));
        return null;
    }).when(clusterAdminClient).state(any(), any());
    defaultMaxDocs = AnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS_PER_SHARD.getDefault(Settings.EMPTY);
}
Also used : ClusterSettings(org.opensearch.common.settings.ClusterSettings) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) ClusterAdminClient(org.opensearch.client.ClusterAdminClient) ClusterStateResponse(org.opensearch.action.admin.cluster.state.ClusterStateResponse) ClusterStateRequest(org.opensearch.action.admin.cluster.state.ClusterStateRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) ClusterService(org.opensearch.cluster.service.ClusterService) ActionListener(org.opensearch.action.ActionListener) ClusterName(org.opensearch.cluster.ClusterName) Client(org.opensearch.client.Client) AdminClient(org.opensearch.client.AdminClient) ClusterAdminClient(org.opensearch.client.ClusterAdminClient) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Settings(org.opensearch.common.settings.Settings) AdminClient(org.opensearch.client.AdminClient) ClusterAdminClient(org.opensearch.client.ClusterAdminClient) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) HashSet(java.util.HashSet)

Example 18 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class UpdateMappingTests method testMissingPrimaryJobShards.

// since SETTING_NUMBER_OF_SHARDS is not there, we skip updating
@SuppressWarnings("unchecked")
public void testMissingPrimaryJobShards() {
    ImmutableOpenMap.Builder<String, Settings> indexToSettings = ImmutableOpenMap.builder();
    Settings jobSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
    indexToSettings.put(ADIndex.JOB.getIndexName(), jobSettings);
    GetSettingsResponse getSettingsResponse = new GetSettingsResponse(indexToSettings.build(), ImmutableOpenMap.of());
    doAnswer(invocation -> {
        ActionListener<GetSettingsResponse> listener = (ActionListener<GetSettingsResponse>) invocation.getArgument(2);
        listener.onResponse(getSettingsResponse);
        return null;
    }).when(client).execute(any(), any(), any());
    adIndices.update();
    verify(indicesAdminClient, never()).updateSettings(any(), any());
}
Also used : ActionListener(org.opensearch.action.ActionListener) GetSettingsResponse(org.opensearch.action.admin.indices.settings.get.GetSettingsResponse) Mockito.anyString(org.mockito.Mockito.anyString) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Settings(org.opensearch.common.settings.Settings)

Example 19 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class NodeStateManagerTests method testSettingUpdateBackOffMin.

public void testSettingUpdateBackOffMin() {
    when(clock.millis()).thenReturn(1000L);
    // In setUp method, we mute after 3 tries
    for (int i = 0; i < 4; i++) {
        stateManager.addPressure(nodeId, adId);
    }
    assertTrue(stateManager.isMuted(nodeId, adId));
    Settings newSettings = Settings.builder().put(AnomalyDetectorSettings.BACKOFF_MINUTES.getKey(), "1m").build();
    Settings.Builder target = Settings.builder();
    clusterSettings.updateDynamicSettings(newSettings, target, Settings.builder(), "test");
    clusterSettings.applySettings(target.build());
    stateManager.addPressure(nodeId, adId);
    // move the clobk by 1000 milliseconds
    // when evaluating isMuted, 62000 - 1000 (last mute time) > 60000, which
    // make isMuted true
    when(clock.millis()).thenReturn(62000L);
    assertTrue(!stateManager.isMuted(nodeId, adId));
}
Also used : AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Settings(org.opensearch.common.settings.Settings)

Example 20 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class NodeStateManagerTests method testSettingUpdateMaxRetry.

public void testSettingUpdateMaxRetry() {
    when(clock.millis()).thenReturn(System.currentTimeMillis());
    stateManager.addPressure(nodeId, adId);
    // In setUp method, we mute after 3 tries
    assertTrue(!stateManager.isMuted(nodeId, adId));
    Settings newSettings = Settings.builder().put(AnomalyDetectorSettings.MAX_RETRY_FOR_UNRESPONSIVE_NODE.getKey(), "1").build();
    Settings.Builder target = Settings.builder();
    clusterSettings.updateDynamicSettings(newSettings, target, Settings.builder(), "test");
    clusterSettings.applySettings(target.build());
    stateManager.addPressure(nodeId, adId);
    // since we have one violation and the max is 1, this is flagged as muted
    assertTrue(stateManager.isMuted(nodeId, adId));
}
Also used : AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Settings(org.opensearch.common.settings.Settings)

Aggregations

Settings (org.opensearch.common.settings.Settings)1933 Test (org.junit.Test)493 IndexSettings (org.opensearch.index.IndexSettings)312 ClusterSettings (org.opensearch.common.settings.ClusterSettings)244 Matchers.containsString (org.hamcrest.Matchers.containsString)234 ArrayList (java.util.ArrayList)198 AuthCredentials (org.opensearch.security.user.AuthCredentials)169 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)163 IOException (java.io.IOException)158 ClusterState (org.opensearch.cluster.ClusterState)155 HashMap (java.util.HashMap)154 DiscoveryNode (org.opensearch.cluster.node.DiscoveryNode)128 Version (org.opensearch.Version)125 SingleClusterTest (org.opensearch.security.test.SingleClusterTest)123 Map (java.util.Map)122 List (java.util.List)118 Path (java.nio.file.Path)116 Index (org.opensearch.index.Index)109 HashSet (java.util.HashSet)107 RestHelper (org.opensearch.security.test.helper.rest.RestHelper)105