Search in sources :

Example 6 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunner method runAnomalyDetectionJob.

private void runAnomalyDetectionJob(AnomalyDetectorJob jobParameter, LockService lockService, LockModel lock, Instant detectionStartTime, Instant executionStartTime, String detectorId, String user, List<String> roles) {
    try (InjectSecurity injectSecurity = new InjectSecurity(detectorId, settings, client.threadPool().getThreadContext())) {
        // Injecting user role to verify if the user has permissions for our API.
        injectSecurity.inject(user, roles);
        AnomalyResultRequest request = new AnomalyResultRequest(detectorId, detectionStartTime.toEpochMilli(), executionStartTime.toEpochMilli());
        client.execute(AnomalyResultAction.INSTANCE, request, ActionListener.wrap(response -> {
            indexAnomalyResult(jobParameter, lockService, lock, detectionStartTime, executionStartTime, response);
        }, exception -> {
            handleAdException(jobParameter, lockService, lock, detectionStartTime, executionStartTime, exception);
        }));
    } catch (Exception e) {
        indexAnomalyResultException(jobParameter, lockService, lock, detectionStartTime, executionStartTime, e, true);
        log.error("Failed to execute AD job " + detectorId, e);
    }
}
Also used : ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) XContentParser(org.opensearch.common.xcontent.XContentParser) AD_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_THREAD_POOL_NAME) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) WriteRequest(org.opensearch.action.support.WriteRequest) ADIndex(org.opensearch.ad.indices.ADIndex) ActionListener(org.opensearch.action.ActionListener) GetResponse(org.opensearch.action.get.GetResponse) Client(org.opensearch.client.Client) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) Set(java.util.Set) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskManager(org.opensearch.ad.task.ADTaskManager) Settings(org.opensearch.common.settings.Settings) Instant(java.time.Instant) UPDATED(org.opensearch.action.DocWriteResponse.Result.UPDATED) List(java.util.List) AnomalyResultAction(org.opensearch.ad.transport.AnomalyResultAction) Logger(org.apache.logging.log4j.Logger) AnomalyResult(org.opensearch.ad.model.AnomalyResult) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) XContentType(org.opensearch.common.xcontent.XContentType) InjectSecurity(org.opensearch.commons.InjectSecurity) JobExecutionContext(org.opensearch.jobscheduler.spi.JobExecutionContext) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) AnomalyIndexHandler(org.opensearch.ad.transport.handler.AnomalyIndexHandler) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) FeatureData(org.opensearch.ad.model.FeatureData) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) ProfileRequest(org.opensearch.ad.transport.ProfileRequest) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) LockModel(org.opensearch.jobscheduler.spi.LockModel) ExecutorService(java.util.concurrent.ExecutorService) EndRunException(org.opensearch.ad.common.exception.EndRunException) LockService(org.opensearch.jobscheduler.spi.utils.LockService) InternalFailure(org.opensearch.ad.common.exception.InternalFailure) GetRequest(org.opensearch.action.get.GetRequest) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) ScheduledJobParameter(org.opensearch.jobscheduler.spi.ScheduledJobParameter) AnomalyResultTransportAction(org.opensearch.ad.transport.AnomalyResultTransportAction) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) User(org.opensearch.commons.authuser.User) AnomalyResultResponse(org.opensearch.ad.transport.AnomalyResultResponse) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ScheduledJobRunner(org.opensearch.jobscheduler.spi.ScheduledJobRunner) IndexRequest(org.opensearch.action.index.IndexRequest) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) LogManager(org.apache.logging.log4j.LogManager) ProfileAction(org.opensearch.ad.transport.ProfileAction) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) InjectSecurity(org.opensearch.commons.InjectSecurity) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) EndRunException(org.opensearch.ad.common.exception.EndRunException) IOException(java.io.IOException)

Example 7 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class ADTaskManagerTests method testCleanADResultOfDeletedDetectorWithException.

public void testCleanADResultOfDeletedDetectorWithException() {
    String detectorId = randomAlphaOfLength(5);
    when(adTaskCacheManager.pollDeletedDetector()).thenReturn(detectorId);
    doAnswer(invocation -> {
        ActionListener<BulkByScrollResponse> listener = invocation.getArgument(2);
        listener.onFailure(new RuntimeException("test"));
        return null;
    }).doAnswer(invocation -> {
        ActionListener<BulkByScrollResponse> listener = invocation.getArgument(2);
        BulkByScrollResponse deleteByQueryResponse = mock(BulkByScrollResponse.class);
        listener.onResponse(deleteByQueryResponse);
        return null;
    }).when(client).execute(any(), any(), any());
    settings = Settings.builder().put(MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.getKey(), 2).put(BATCH_TASK_PIECE_INTERVAL_SECONDS.getKey(), 1).put(REQUEST_TIMEOUT.getKey(), TimeValue.timeValueSeconds(10)).put(DELETE_AD_RESULT_WHEN_DELETE_DETECTOR.getKey(), true).build();
    clusterSettings = clusterSetting(settings, MAX_OLD_AD_TASK_DOCS_PER_DETECTOR, BATCH_TASK_PIECE_INTERVAL_SECONDS, REQUEST_TIMEOUT, DELETE_AD_RESULT_WHEN_DELETE_DETECTOR, MAX_BATCH_TASK_PER_NODE, MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS);
    clusterService = spy(new ClusterService(settings, clusterSettings, null));
    ADTaskManager adTaskManager = spy(new ADTaskManager(settings, clusterService, client, TestHelpers.xContentRegistry(), detectionIndices, nodeFilter, hashRing, adTaskCacheManager, threadPool));
    adTaskManager.cleanADResultOfDeletedDetector();
    verify(client, times(1)).execute(eq(DeleteByQueryAction.INSTANCE), any(), any());
    verify(adTaskCacheManager, times(1)).addDeletedDetector(eq(detectorId));
    adTaskManager.cleanADResultOfDeletedDetector();
    verify(client, times(2)).execute(eq(DeleteByQueryAction.INSTANCE), any(), any());
    verify(adTaskCacheManager, times(1)).addDeletedDetector(eq(detectorId));
}
Also used : ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) IndexResponse(org.opensearch.action.index.IndexResponse) HashRing(org.opensearch.ad.cluster.HashRing) Version(org.opensearch.Version) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) Mockito.doThrow(org.mockito.Mockito.doThrow) ADUnitTestCase(org.opensearch.ad.ADUnitTestCase) Mockito.doAnswer(org.mockito.Mockito.doAnswer) TestHelpers.randomIntervalSchedule(org.opensearch.ad.TestHelpers.randomIntervalSchedule) MAX_OLD_AD_TASK_DOCS_PER_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR) Map(java.util.Map) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) ActionListener(org.opensearch.action.ActionListener) Mockito.doReturn(org.mockito.Mockito.doReturn) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) Index(org.opensearch.index.Index) SearchHit(org.opensearch.search.SearchHit) GetResult(org.opensearch.index.get.GetResult) Settings(org.opensearch.common.settings.Settings) DETECTION_STATE_INDEX(org.opensearch.ad.constant.CommonName.DETECTION_STATE_INDEX) TransportService(org.opensearch.transport.TransportService) RandomCutForest(com.amazon.randomcutforest.RandomCutForest) ArgumentMatchers.anyFloat(org.mockito.ArgumentMatchers.anyFloat) TestHelpers(org.opensearch.ad.TestHelpers) DocWriteResponse(org.opensearch.action.DocWriteResponse) Mockito.mock(org.mockito.Mockito.mock) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) TestHelpers.randomFeature(org.opensearch.ad.TestHelpers.randomFeature) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) ADTaskProfileResponse(org.opensearch.ad.transport.ADTaskProfileResponse) DocWriteRequest(org.opensearch.action.DocWriteRequest) ArgumentMatchers.anyMap(org.mockito.ArgumentMatchers.anyMap) TestHelpers.randomDetectionDateRange(org.opensearch.ad.TestHelpers.randomDetectionDateRange) SearchHits(org.opensearch.search.SearchHits) Mockito.spy(org.mockito.Mockito.spy) ADTaskProfile(org.opensearch.ad.model.ADTaskProfile) TestHelpers.randomAdTask(org.opensearch.ad.TestHelpers.randomAdTask) ArrayList(java.util.ArrayList) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) DeleteResponse(org.opensearch.action.delete.DeleteResponse) SearchRequest(org.opensearch.action.search.SearchRequest) CommonName(org.opensearch.ad.constant.CommonName) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) TotalHits(org.apache.lucene.search.TotalHits) Mockito.never(org.mockito.Mockito.never) ChronoUnit(java.time.temporal.ChronoUnit) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) LinkedBlockingDeque(java.util.concurrent.LinkedBlockingDeque) ClusterService(org.opensearch.cluster.service.ClusterService) ADTaskAction(org.opensearch.ad.model.ADTaskAction) DeleteByQueryAction(org.opensearch.index.reindex.DeleteByQueryAction) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) DELETE_AD_RESULT_WHEN_DELETE_DETECTOR(org.opensearch.ad.settings.AnomalyDetectorSettings.DELETE_AD_RESULT_WHEN_DELETE_DETECTOR) ToXContent(org.opensearch.common.xcontent.ToXContent) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) TestHelpers.randomAnomalyDetector(org.opensearch.ad.TestHelpers.randomAnomalyDetector) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) MapperService(org.opensearch.index.mapper.MapperService) IndexAnomalyDetectorJobActionHandler(org.opensearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler) Locale(java.util.Locale) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) TestHelpers.randomDetector(org.opensearch.ad.TestHelpers.randomDetector) GetResponse(org.opensearch.action.get.GetResponse) InternalStatNames(org.opensearch.ad.stats.InternalStatNames) TestHelpers.randomIntervalTimeConfiguration(org.opensearch.ad.TestHelpers.randomIntervalTimeConfiguration) ImmutableMap(com.google.common.collect.ImmutableMap) ADTaskType(org.opensearch.ad.model.ADTaskType) ADTaskProfileNodeResponse(org.opensearch.ad.transport.ADTaskProfileNodeResponse) Instant(java.time.Instant) TransportAddress(org.opensearch.common.transport.TransportAddress) AnomalyDetectorJobResponse(org.opensearch.ad.transport.AnomalyDetectorJobResponse) List(java.util.List) ForwardADTaskRequest(org.opensearch.ad.transport.ForwardADTaskRequest) DuplicateTaskException(org.opensearch.ad.common.exception.DuplicateTaskException) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) Optional(java.util.Optional) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) ANOMALY_RESULT_INDEX_ALIAS(org.opensearch.ad.constant.CommonName.ANOMALY_RESULT_INDEX_ALIAS) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) REQUEST_TIMEOUT(org.opensearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT) ADStatsNodesResponse(org.opensearch.ad.transport.ADStatsNodesResponse) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) BytesReference(org.opensearch.common.bytes.BytesReference) ADTask(org.opensearch.ad.model.ADTask) TestHelpers.randomUser(org.opensearch.ad.TestHelpers.randomUser) Deque(java.util.Deque) Captor(org.mockito.Captor) ArgumentCaptor(org.mockito.ArgumentCaptor) ImmutableList(com.google.common.collect.ImmutableList) SearchResponse(org.opensearch.action.search.SearchResponse) ClusterSettings(org.opensearch.common.settings.ClusterSettings) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) ExecutorService(java.util.concurrent.ExecutorService) Collections.emptyMap(java.util.Collections.emptyMap) Collections.emptySet(java.util.Collections.emptySet) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) Entity.createSingleAttributeEntity(org.opensearch.ad.model.Entity.createSingleAttributeEntity) MockSimpleLog(org.opensearch.ad.mock.model.MockSimpleLog) TransportResponseHandler(org.opensearch.transport.TransportResponseHandler) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) Mockito.when(org.mockito.Mockito.when) CreateIndexResponse(org.opensearch.action.admin.indices.create.CreateIndexResponse) Mockito.verify(org.mockito.Mockito.verify) CREATE_INDEX_NOT_ACKNOWLEDGED(org.opensearch.ad.constant.CommonErrorMessages.CREATE_INDEX_NOT_ACKNOWLEDGED) Consumer(java.util.function.Consumer) ShardId(org.opensearch.index.shard.ShardId) BulkByScrollResponse(org.opensearch.index.reindex.BulkByScrollResponse) Entity(org.opensearch.ad.model.Entity) BulkResponse(org.opensearch.action.bulk.BulkResponse) User(org.opensearch.commons.authuser.User) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) ThresholdedRandomCutForest(com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest) ClusterName(org.opensearch.cluster.ClusterName) Collections(java.util.Collections) ClusterService(org.opensearch.cluster.service.ClusterService) ActionListener(org.opensearch.action.ActionListener) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) BulkByScrollResponse(org.opensearch.index.reindex.BulkByScrollResponse)

Example 8 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class AnomalyDetectorSettingsTests method testSettingsGetValue.

public void testSettingsGetValue() {
    Settings settings = Settings.builder().put("plugins.anomaly_detection.request_timeout", "42s").build();
    assertEquals(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(42));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(10));
    settings = Settings.builder().put("plugins.anomaly_detection.max_anomaly_detectors", 99).build();
    assertEquals(AnomalyDetectorSettings.MAX_SINGLE_ENTITY_ANOMALY_DETECTORS.get(settings), Integer.valueOf(99));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_SINGLE_ENTITY_ANOMALY_DETECTORS.get(settings), Integer.valueOf(1000));
    settings = Settings.builder().put("plugins.anomaly_detection.max_multi_entity_anomaly_detectors", 98).build();
    assertEquals(AnomalyDetectorSettings.MAX_MULTI_ENTITY_ANOMALY_DETECTORS.get(settings), Integer.valueOf(98));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_MULTI_ENTITY_ANOMALY_DETECTORS.get(settings), Integer.valueOf(10));
    settings = Settings.builder().put("plugins.anomaly_detection.max_anomaly_features", 7).build();
    assertEquals(AnomalyDetectorSettings.MAX_ANOMALY_FEATURES.get(settings), Integer.valueOf(7));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_ANOMALY_FEATURES.get(settings), Integer.valueOf(5));
    settings = Settings.builder().put("plugins.anomaly_detection.detection_interval", TimeValue.timeValueMinutes(96)).build();
    assertEquals(AnomalyDetectorSettings.DETECTION_INTERVAL.get(settings), TimeValue.timeValueMinutes(96));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.DETECTION_INTERVAL.get(settings), TimeValue.timeValueMinutes(10));
    settings = Settings.builder().put("plugins.anomaly_detection.detection_window_delay", TimeValue.timeValueMinutes(95)).build();
    assertEquals(AnomalyDetectorSettings.DETECTION_WINDOW_DELAY.get(settings), TimeValue.timeValueMinutes(95));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.DETECTION_WINDOW_DELAY.get(settings), TimeValue.timeValueMinutes(0));
    settings = Settings.builder().put("plugins.anomaly_detection.ad_result_history_rollover_period", TimeValue.timeValueHours(94)).build();
    assertEquals(AnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD.get(settings), TimeValue.timeValueHours(94));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD.get(settings), TimeValue.timeValueHours(12));
    settings = Settings.builder().put("plugins.anomaly_detection.ad_result_history_max_docs_per_shard", 93).build();
    assertEquals(AnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS_PER_SHARD.get(settings), Long.valueOf(93));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS.get(settings), Long.valueOf(250000000));
    settings = Settings.builder().put("plugins.anomaly_detection.ad_result_history_retention_period", TimeValue.timeValueDays(92)).build();
    assertEquals(AnomalyDetectorSettings.AD_RESULT_HISTORY_RETENTION_PERIOD.get(settings), TimeValue.timeValueDays(92));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_RETENTION_PERIOD.get(settings), TimeValue.timeValueDays(30));
    settings = Settings.builder().put("plugins.anomaly_detection.max_retry_for_unresponsive_node", 91).build();
    assertEquals(AnomalyDetectorSettings.MAX_RETRY_FOR_UNRESPONSIVE_NODE.get(settings), Integer.valueOf(91));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_RETRY_FOR_UNRESPONSIVE_NODE.get(settings), Integer.valueOf(5));
    settings = Settings.builder().put("plugins.anomaly_detection.cooldown_minutes", TimeValue.timeValueMinutes(90)).build();
    assertEquals(AnomalyDetectorSettings.COOLDOWN_MINUTES.get(settings), TimeValue.timeValueMinutes(90));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.COOLDOWN_MINUTES.get(settings), TimeValue.timeValueMinutes(5));
    settings = Settings.builder().put("plugins.anomaly_detection.backoff_minutes", TimeValue.timeValueMinutes(89)).build();
    assertEquals(AnomalyDetectorSettings.BACKOFF_MINUTES.get(settings), TimeValue.timeValueMinutes(89));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.BACKOFF_MINUTES.get(settings), TimeValue.timeValueMinutes(15));
    settings = Settings.builder().put("plugins.anomaly_detection.backoff_initial_delay", TimeValue.timeValueMillis(88)).build();
    assertEquals(AnomalyDetectorSettings.BACKOFF_INITIAL_DELAY.get(settings), TimeValue.timeValueMillis(88));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.BACKOFF_INITIAL_DELAY.get(settings), TimeValue.timeValueMillis(1000));
    settings = Settings.builder().put("plugins.anomaly_detection.max_retry_for_backoff", 87).build();
    assertEquals(AnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF.get(settings), Integer.valueOf(87));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF.get(settings), Integer.valueOf(3));
    settings = Settings.builder().put("plugins.anomaly_detection.max_retry_for_end_run_exception", 86).build();
    assertEquals(AnomalyDetectorSettings.MAX_RETRY_FOR_END_RUN_EXCEPTION.get(settings), Integer.valueOf(86));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_RETRY_FOR_END_RUN_EXCEPTION.get(settings), Integer.valueOf(6));
    settings = Settings.builder().put("plugins.anomaly_detection.filter_by_backend_roles", true).build();
    assertEquals(AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES.get(settings), Boolean.valueOf(true));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES.get(settings), Boolean.valueOf(false));
    settings = Settings.builder().put("plugins.anomaly_detection.model_max_size_percent", 0.3).build();
    assertEquals(AnomalyDetectorSettings.MODEL_MAX_SIZE_PERCENTAGE.get(settings), Double.valueOf(0.3));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MODEL_MAX_SIZE_PERCENTAGE.get(settings), Double.valueOf(0.1));
    settings = Settings.builder().put("plugins.anomaly_detection.max_entities_per_query", 83).build();
    assertEquals(AnomalyDetectorSettings.MAX_ENTITIES_PER_QUERY.get(settings), Integer.valueOf(83));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_ENTITIES_PER_QUERY.get(settings), Integer.valueOf(1000));
    settings = Settings.builder().put("plugins.anomaly_detection.max_entities_for_preview", 22).build();
    assertEquals(AnomalyDetectorSettings.MAX_ENTITIES_FOR_PREVIEW.get(settings), Integer.valueOf(22));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_ENTITIES_FOR_PREVIEW.get(settings), Integer.valueOf(30));
    settings = Settings.builder().put("plugins.anomaly_detection.index_pressure_soft_limit", 81f).build();
    assertEquals(AnomalyDetectorSettings.INDEX_PRESSURE_SOFT_LIMIT.get(settings), Float.valueOf(81f));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.INDEX_PRESSURE_SOFT_LIMIT.get(settings), Float.valueOf(0.8f));
    settings = Settings.builder().put("plugins.anomaly_detection.max_primary_shards", 80).build();
    assertEquals(AnomalyDetectorSettings.MAX_PRIMARY_SHARDS.get(settings), Integer.valueOf(80));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_PRIMARY_SHARDS.get(settings), Integer.valueOf(10));
    settings = Settings.builder().put("plugins.anomaly_detection.max_cache_miss_handling_per_second", 79).build();
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_CACHE_MISS_HANDLING_PER_SECOND.get(settings), Integer.valueOf(100));
    settings = Settings.builder().put("plugins.anomaly_detection.max_batch_task_per_node", 78).build();
    assertEquals(AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE.get(settings), Integer.valueOf(78));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE.get(settings), Integer.valueOf(10));
    settings = Settings.builder().put("plugins.anomaly_detection.max_old_ad_task_docs_per_detector", 77).build();
    assertEquals(AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.get(settings), Integer.valueOf(77));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.get(settings), Integer.valueOf(1));
    settings = Settings.builder().put("plugins.anomaly_detection.batch_task_piece_size", 76).build();
    assertEquals(AnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE.get(settings), Integer.valueOf(76));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE.get(settings), Integer.valueOf(1000));
    settings = Settings.builder().put("plugins.anomaly_detection.batch_task_piece_interval_seconds", 76).build();
    assertEquals(AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS.get(settings), Integer.valueOf(76));
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS.get(settings), Integer.valueOf(5));
}
Also used : Settings(org.opensearch.common.settings.Settings)

Example 9 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class AnomalyDetectorSettingsTests method testSettingsGetValueWithLegacyFallback.

public void testSettingsGetValueWithLegacyFallback() {
    Settings settings = Settings.builder().put("opendistro.anomaly_detection.max_anomaly_detectors", 1).put("opendistro.anomaly_detection.max_multi_entity_anomaly_detectors", 2).put("opendistro.anomaly_detection.max_anomaly_features", 3).put("opendistro.anomaly_detection.request_timeout", "4s").put("opendistro.anomaly_detection.detection_interval", "5m").put("opendistro.anomaly_detection.detection_window_delay", "6m").put("opendistro.anomaly_detection.ad_result_history_rollover_period", "7h").put("opendistro.anomaly_detection.ad_result_history_max_docs", 8L).put("opendistro.anomaly_detection.ad_result_history_retention_period", "9d").put("opendistro.anomaly_detection.max_retry_for_unresponsive_node", 10).put("opendistro.anomaly_detection.cooldown_minutes", "11m").put("opendistro.anomaly_detection.backoff_minutes", "12m").put("opendistro.anomaly_detection.backoff_initial_delay", // 
    "13ms").put("opendistro.anomaly_detection.max_retry_for_backoff", 14).put("opendistro.anomaly_detection.max_retry_for_end_run_exception", 15).put("opendistro.anomaly_detection.filter_by_backend_roles", true).put("opendistro.anomaly_detection.model_max_size_percent", 0.6D).put("opendistro.anomaly_detection.max_entities_for_preview", 19).put("opendistro.anomaly_detection.index_pressure_soft_limit", 20F).put("opendistro.anomaly_detection.max_primary_shards", 21).put("opendistro.anomaly_detection.max_cache_miss_handling_per_second", 22).put("opendistro.anomaly_detection.max_batch_task_per_node", 23).put("opendistro.anomaly_detection.max_old_ad_task_docs_per_detector", 24).put("opendistro.anomaly_detection.batch_task_piece_size", 25).put("opendistro.anomaly_detection.batch_task_piece_interval_seconds", 26).build();
    assertEquals(AnomalyDetectorSettings.MAX_SINGLE_ENTITY_ANOMALY_DETECTORS.get(settings), Integer.valueOf(1));
    assertEquals(AnomalyDetectorSettings.MAX_MULTI_ENTITY_ANOMALY_DETECTORS.get(settings), Integer.valueOf(2));
    assertEquals(AnomalyDetectorSettings.MAX_ANOMALY_FEATURES.get(settings), Integer.valueOf(3));
    assertEquals(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(settings), TimeValue.timeValueSeconds(4));
    assertEquals(AnomalyDetectorSettings.DETECTION_INTERVAL.get(settings), TimeValue.timeValueMinutes(5));
    assertEquals(AnomalyDetectorSettings.DETECTION_WINDOW_DELAY.get(settings), TimeValue.timeValueMinutes(6));
    assertEquals(AnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD.get(settings), TimeValue.timeValueHours(7));
    // AD_RESULT_HISTORY_MAX_DOCS is removed in the new release
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS.get(settings), Long.valueOf(8L));
    assertEquals(AnomalyDetectorSettings.AD_RESULT_HISTORY_RETENTION_PERIOD.get(settings), TimeValue.timeValueDays(9));
    assertEquals(AnomalyDetectorSettings.MAX_RETRY_FOR_UNRESPONSIVE_NODE.get(settings), Integer.valueOf(10));
    assertEquals(AnomalyDetectorSettings.COOLDOWN_MINUTES.get(settings), TimeValue.timeValueMinutes(11));
    assertEquals(AnomalyDetectorSettings.BACKOFF_MINUTES.get(settings), TimeValue.timeValueMinutes(12));
    assertEquals(AnomalyDetectorSettings.BACKOFF_INITIAL_DELAY.get(settings), TimeValue.timeValueMillis(13));
    assertEquals(AnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF.get(settings), Integer.valueOf(14));
    assertEquals(AnomalyDetectorSettings.MAX_RETRY_FOR_END_RUN_EXCEPTION.get(settings), Integer.valueOf(15));
    assertEquals(AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES.get(settings), Boolean.valueOf(true));
    assertEquals(AnomalyDetectorSettings.MODEL_MAX_SIZE_PERCENTAGE.get(settings), Double.valueOf(0.6D));
    // MAX_ENTITIES_FOR_PREVIEW uses default instead of legacy fallback
    assertEquals(AnomalyDetectorSettings.MAX_ENTITIES_FOR_PREVIEW.get(settings), Integer.valueOf(5));
    // INDEX_PRESSURE_SOFT_LIMIT uses default instead of legacy fallback
    assertEquals(AnomalyDetectorSettings.INDEX_PRESSURE_SOFT_LIMIT.get(settings), Float.valueOf(0.6F));
    assertEquals(AnomalyDetectorSettings.MAX_PRIMARY_SHARDS.get(settings), Integer.valueOf(21));
    // MAX_CACHE_MISS_HANDLING_PER_SECOND is removed in the new release
    assertEquals(LegacyOpenDistroAnomalyDetectorSettings.MAX_CACHE_MISS_HANDLING_PER_SECOND.get(settings), Integer.valueOf(22));
    assertEquals(AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE.get(settings), Integer.valueOf(23));
    assertEquals(AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.get(settings), Integer.valueOf(24));
    assertEquals(AnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE.get(settings), Integer.valueOf(25));
    assertEquals(AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS.get(settings), Integer.valueOf(26));
    assertSettingDeprecationsAndWarnings(new Setting[] { LegacyOpenDistroAnomalyDetectorSettings.MAX_SINGLE_ENTITY_ANOMALY_DETECTORS, LegacyOpenDistroAnomalyDetectorSettings.MAX_MULTI_ENTITY_ANOMALY_DETECTORS, LegacyOpenDistroAnomalyDetectorSettings.MAX_ANOMALY_FEATURES, LegacyOpenDistroAnomalyDetectorSettings.REQUEST_TIMEOUT, LegacyOpenDistroAnomalyDetectorSettings.DETECTION_INTERVAL, LegacyOpenDistroAnomalyDetectorSettings.DETECTION_WINDOW_DELAY, LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD, LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS, LegacyOpenDistroAnomalyDetectorSettings.MAX_RETRY_FOR_UNRESPONSIVE_NODE, LegacyOpenDistroAnomalyDetectorSettings.COOLDOWN_MINUTES, LegacyOpenDistroAnomalyDetectorSettings.BACKOFF_MINUTES, LegacyOpenDistroAnomalyDetectorSettings.BACKOFF_INITIAL_DELAY, LegacyOpenDistroAnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF, LegacyOpenDistroAnomalyDetectorSettings.MAX_RETRY_FOR_END_RUN_EXCEPTION, LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_RETENTION_PERIOD, LegacyOpenDistroAnomalyDetectorSettings.MODEL_MAX_SIZE_PERCENTAGE, LegacyOpenDistroAnomalyDetectorSettings.MAX_PRIMARY_SHARDS, LegacyOpenDistroAnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES, LegacyOpenDistroAnomalyDetectorSettings.MAX_CACHE_MISS_HANDLING_PER_SECOND, LegacyOpenDistroAnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE, LegacyOpenDistroAnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS, LegacyOpenDistroAnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR, LegacyOpenDistroAnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE });
}
Also used : Settings(org.opensearch.common.settings.Settings)

Example 10 with Settings

use of org.opensearch.common.settings.Settings in project anomaly-detection by opensearch-project.

the class ADStatsTests method setup.

@Before
public void setup() {
    MockitoAnnotations.initMocks(this);
    // sampleSize * numberOfTrees has to be larger than 1. Otherwise, RCF reports errors.
    rcf = RandomCutForest.builder().dimensions(1).sampleSize(2).numberOfTrees(1).build();
    thresholdingModel = new HybridThresholdingModel(1e-8, 1e-5, 200, 10_000, 2, 5_000_000);
    List<ModelState<?>> modelsInformation = new ArrayList<>(Arrays.asList(new ModelState<>(rcf, "rcf-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-1", "detector-1", ModelManager.ModelType.RCF.getName(), clock, 0f), new ModelState<>(rcf, "rcf-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f), new ModelState<>(thresholdingModel, "thr-model-2", "detector-2", ModelManager.ModelType.THRESHOLD.getName(), clock, 0f)));
    when(modelManager.getAllModels()).thenReturn(modelsInformation);
    ModelState<EntityModel> entityModel1 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    ModelState<EntityModel> entityModel2 = MLUtil.randomModelState(new RandomModelStateConfig.Builder().fullModel(true).build());
    List<ModelState<?>> entityModelsInformation = new ArrayList<>(Arrays.asList(entityModel1, entityModel2));
    EntityCache cache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getAllModels()).thenReturn(entityModelsInformation);
    IndexUtils indexUtils = mock(IndexUtils.class);
    when(indexUtils.getIndexHealthStatus(anyString())).thenReturn("yellow");
    when(indexUtils.getNumberOfDocumentsInIndex(anyString())).thenReturn(100L);
    clusterStatName1 = "clusterStat1";
    clusterStatName2 = "clusterStat2";
    nodeStatName1 = "nodeStat1";
    nodeStatName2 = "nodeStat2";
    Settings settings = Settings.builder().put(MAX_MODEL_SIZE_PER_NODE.getKey(), 10).build();
    ClusterService clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(MAX_MODEL_SIZE_PER_NODE))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    statsMap = new HashMap<String, ADStat<?>>() {

        {
            put(nodeStatName1, new ADStat<>(false, new CounterSupplier()));
            put(nodeStatName2, new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider, settings, clusterService)));
            put(clusterStatName1, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index1")));
            put(clusterStatName2, new ADStat<>(true, new IndexStatusSupplier(indexUtils, "index2")));
        }
    };
    adStats = new ADStats(statsMap);
}
Also used : ModelsOnNodeSupplier(org.opensearch.ad.stats.suppliers.ModelsOnNodeSupplier) ClusterSettings(org.opensearch.common.settings.ClusterSettings) EntityCache(org.opensearch.ad.caching.EntityCache) ArrayList(java.util.ArrayList) EntityModel(org.opensearch.ad.ml.EntityModel) HybridThresholdingModel(org.opensearch.ad.ml.HybridThresholdingModel) ModelState(org.opensearch.ad.ml.ModelState) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) RandomModelStateConfig(test.org.opensearch.ad.util.RandomModelStateConfig) CounterSupplier(org.opensearch.ad.stats.suppliers.CounterSupplier) IndexUtils(org.opensearch.ad.util.IndexUtils) ClusterService(org.opensearch.cluster.service.ClusterService) IndexStatusSupplier(org.opensearch.ad.stats.suppliers.IndexStatusSupplier) ClusterSettings(org.opensearch.common.settings.ClusterSettings) Settings(org.opensearch.common.settings.Settings) HashSet(java.util.HashSet) Before(org.junit.Before)

Aggregations

Settings (org.opensearch.common.settings.Settings)1933 Test (org.junit.Test)493 IndexSettings (org.opensearch.index.IndexSettings)312 ClusterSettings (org.opensearch.common.settings.ClusterSettings)244 Matchers.containsString (org.hamcrest.Matchers.containsString)234 ArrayList (java.util.ArrayList)198 AuthCredentials (org.opensearch.security.user.AuthCredentials)169 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)163 IOException (java.io.IOException)158 ClusterState (org.opensearch.cluster.ClusterState)155 HashMap (java.util.HashMap)154 DiscoveryNode (org.opensearch.cluster.node.DiscoveryNode)128 Version (org.opensearch.Version)125 SingleClusterTest (org.opensearch.security.test.SingleClusterTest)123 Map (java.util.Map)122 List (java.util.List)118 Path (java.nio.file.Path)116 Index (org.opensearch.index.Index)109 HashSet (java.util.HashSet)107 RestHelper (org.opensearch.security.test.helper.rest.RestHelper)105