Examples with FeatureManager org.opensearch.ad.feature.FeatureManager used on opensource projects

Example 1 with FeatureManager

use of org.opensearch.ad.feature.FeatureManager in project anomaly-detection by opensearch-project.

the class EntityColdStarterTests method setUp.

@SuppressWarnings("unchecked")
@Override
public void setUp() throws Exception {
    super.setUp();
    numMinSamples = AnomalyDetectorSettings.NUM_MIN_SAMPLES;
    clock = mock(Clock.class);
    when(clock.instant()).thenReturn(Instant.now());
    threadPool = mock(ThreadPool.class);
    setUpADThreadPool(threadPool);
    settings = Settings.EMPTY;
    Client client = mock(Client.class);
    clientUtil = mock(ClientUtil.class);
    detector = TestHelpers.AnomalyDetectorBuilder.newInstance().setDetectionInterval(new IntervalTimeConfiguration(1, ChronoUnit.MINUTES)).setCategoryFields(ImmutableList.of(randomAlphaOfLength(5))).build();
    job = TestHelpers.randomAnomalyDetectorJob(true, Instant.ofEpochMilli(1602401500000L), null);
    doAnswer(invocation -> {
        GetRequest request = invocation.getArgument(0);
        ActionListener<GetResponse> listener = invocation.getArgument(2);
        if (request.index().equals(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(job, detectorId, AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX));
        } else {
            listener.onResponse(TestHelpers.createGetResponse(detector, detectorId, AnomalyDetector.ANOMALY_DETECTORS_INDEX));
        }
        return null;
    }).when(clientUtil).asyncRequest(any(GetRequest.class), any(), any(ActionListener.class));
    Set<Setting<?>> nodestateSetting = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    nodestateSetting.add(MAX_RETRY_FOR_UNRESPONSIVE_NODE);
    nodestateSetting.add(BACKOFF_MINUTES);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, nodestateSetting);
    DiscoveryNode discoveryNode = new DiscoveryNode("node1", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
    ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, clusterSettings);
    stateManager = new NodeStateManager(client, xContentRegistry(), settings, clientUtil, clock, AnomalyDetectorSettings.HOURLY_MAINTENANCE, clusterService);
    SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = new IntegerSensitiveSingleFeatureLinearUniformInterpolator();
    interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator);
    searchFeatureDao = mock(SearchFeatureDao.class);
    checkpoint = mock(CheckpointDao.class);
    featureManager = new FeatureManager(searchFeatureDao, interpolator, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.TRAIN_SAMPLE_TIME_RANGE_IN_HOURS, AnomalyDetectorSettings.MIN_TRAIN_SAMPLES, AnomalyDetectorSettings.MAX_SHINGLE_PROPORTION_MISSING, AnomalyDetectorSettings.MAX_IMPUTATION_NEIGHBOR_DISTANCE, AnomalyDetectorSettings.PREVIEW_SAMPLE_RATE, AnomalyDetectorSettings.MAX_PREVIEW_SAMPLES, AnomalyDetectorSettings.HOURLY_MAINTENANCE, threadPool, AnomalyDetectorPlugin.AD_THREAD_POOL_NAME);
    checkpointWriteQueue = mock(CheckpointWriteWorker.class);
    rcfSeed = 2051L;
    entityColdStarter = new EntityColdStarter(clock, threadPool, stateManager, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.TIME_DECAY, numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, interpolator, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, settings, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue, rcfSeed, AnomalyDetectorSettings.MAX_COLD_START_ROUNDS);
    detectorId = "123";
    modelId = "123_entity_abc";
    entityName = "abc";
    priority = 0.3f;
    entity = Entity.createSingleAttributeEntity("field", entityName);
    released = new AtomicBoolean();
    inProgressLatch = new CountDownLatch(1);
    releaseSemaphore = () -> {
        released.set(true);
        inProgressLatch.countDown();
    };
    listener = ActionListener.wrap(releaseSemaphore);
    modelManager = new ModelManager(mock(CheckpointDao.class), mock(Clock.class), AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.TIME_DECAY, AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, AnomalyDetectorSettings.MIN_PREVIEW_SIZE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, entityColdStarter, mock(FeatureManager.class), mock(MemoryTracker.class));
}

Example 2 with FeatureManager

use of org.opensearch.ad.feature.FeatureManager in project anomaly-detection by opensearch-project.

the class PreviewAnomalyDetectorTransportActionTests method setUp.

@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    task = mock(Task.class);
    clusterService = mock(ClusterService.class);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AnomalyDetectorSettings.MAX_ANOMALY_FEATURES, AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES, AnomalyDetectorSettings.PAGE_SIZE, AnomalyDetectorSettings.MAX_CONCURRENT_PREVIEW))));
    when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
    ClusterName clusterName = new ClusterName("test");
    Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
    final Settings.Builder existingSettings = Settings.builder().put(indexSettings).put(IndexMetadata.SETTING_INDEX_UUID, "test2UUID");
    IndexMetadata indexMetaData = IndexMetadata.builder(AnomalyDetector.ANOMALY_DETECTORS_INDEX).settings(existingSettings).build();
    final ImmutableOpenMap<String, IndexMetadata> indices = ImmutableOpenMap.<String, IndexMetadata>builder().fPut(AnomalyDetector.ANOMALY_DETECTORS_INDEX, indexMetaData).build();
    ClusterState clusterState = ClusterState.builder(clusterName).metadata(Metadata.builder().indices(indices).build()).build();
    when(clusterService.state()).thenReturn(clusterState);
    featureManager = mock(FeatureManager.class);
    modelManager = mock(ModelManager.class);
    runner = new AnomalyDetectorRunner(modelManager, featureManager, AnomalyDetectorSettings.MAX_PREVIEW_RESULTS);
    circuitBreaker = mock(ADCircuitBreakerService.class);
    when(circuitBreaker.isOpen()).thenReturn(false);
    action = new PreviewAnomalyDetectorTransportAction(Settings.EMPTY, mock(TransportService.class), clusterService, mock(ActionFilters.class), client(), runner, xContentRegistry(), circuitBreaker);
}

Example 3 with FeatureManager

use of org.opensearch.ad.feature.FeatureManager in project anomaly-detection by opensearch-project.

the class DeleteModelTransportActionTests method setUp.

@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    ThreadPool threadPool = mock(ThreadPool.class);
    ClusterService clusterService = mock(ClusterService.class);
    localNodeID = "foo";
    when(clusterService.localNode()).thenReturn(new DiscoveryNode(localNodeID, buildNewFakeTransportAddress(), Version.CURRENT));
    when(clusterService.getClusterName()).thenReturn(new ClusterName("test"));
    TransportService transportService = mock(TransportService.class);
    ActionFilters actionFilters = mock(ActionFilters.class);
    NodeStateManager nodeStateManager = mock(NodeStateManager.class);
    ModelManager modelManager = mock(ModelManager.class);
    FeatureManager featureManager = mock(FeatureManager.class);
    CacheProvider cacheProvider = mock(CacheProvider.class);
    EntityCache entityCache = mock(EntityCache.class);
    when(cacheProvider.get()).thenReturn(entityCache);
    ADTaskCacheManager adTaskCacheManager = mock(ADTaskCacheManager.class);
    NodeStateManager stateManager = mock(NodeStateManager.class);
    action = new DeleteModelTransportAction(threadPool, clusterService, transportService, actionFilters, nodeStateManager, modelManager, featureManager, cacheProvider, adTaskCacheManager);
}

Example 4 with FeatureManager

use of org.opensearch.ad.feature.FeatureManager in project anomaly-detection by opensearch-project.

the class AnomalyDetectorPlugin method createComponents.

@Override
public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<RepositoriesService> repositoriesServiceSupplier) {
    EnabledSetting.getInstance().init(clusterService);
    NumericSetting.getInstance().init(clusterService);
    this.client = client;
    this.threadPool = threadPool;
    Settings settings = environment.settings();
    Throttler throttler = new Throttler(getClock());
    this.clientUtil = new ClientUtil(settings, client, throttler, threadPool);
    this.indexUtils = new IndexUtils(client, clientUtil, clusterService, indexNameExpressionResolver);
    this.nodeFilter = new DiscoveryNodeFilterer(clusterService);
    this.anomalyDetectionIndices = new AnomalyDetectionIndices(client, clusterService, threadPool, settings, nodeFilter, AnomalyDetectorSettings.MAX_UPDATE_RETRY_TIMES);
    this.clusterService = clusterService;
    SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = new IntegerSensitiveSingleFeatureLinearUniformInterpolator();
    Interpolator interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator);
    SearchFeatureDao searchFeatureDao = new SearchFeatureDao(client, xContentRegistry, interpolator, clientUtil, settings, clusterService, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE);
    JvmService jvmService = new JvmService(environment.settings());
    RandomCutForestMapper mapper = new RandomCutForestMapper();
    mapper.setSaveExecutorContextEnabled(true);
    mapper.setSaveTreeStateEnabled(true);
    mapper.setPartialTreeStateEnabled(true);
    V1JsonToV2StateConverter converter = new V1JsonToV2StateConverter();
    double modelMaxSizePercent = AnomalyDetectorSettings.MODEL_MAX_SIZE_PERCENTAGE.get(settings);
    ADCircuitBreakerService adCircuitBreakerService = new ADCircuitBreakerService(jvmService).init();
    MemoryTracker memoryTracker = new MemoryTracker(jvmService, modelMaxSizePercent, AnomalyDetectorSettings.DESIRED_MODEL_SIZE_PERCENTAGE, clusterService, adCircuitBreakerService);
    NodeStateManager stateManager = new NodeStateManager(client, xContentRegistry, settings, clientUtil, getClock(), AnomalyDetectorSettings.HOURLY_MAINTENANCE, clusterService);
    FeatureManager featureManager = new FeatureManager(searchFeatureDao, interpolator, getClock(), AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.TRAIN_SAMPLE_TIME_RANGE_IN_HOURS, AnomalyDetectorSettings.MIN_TRAIN_SAMPLES, AnomalyDetectorSettings.MAX_SHINGLE_PROPORTION_MISSING, AnomalyDetectorSettings.MAX_IMPUTATION_NEIGHBOR_DISTANCE, AnomalyDetectorSettings.PREVIEW_SAMPLE_RATE, AnomalyDetectorSettings.MAX_PREVIEW_SAMPLES, AnomalyDetectorSettings.HOURLY_MAINTENANCE, threadPool, AD_THREAD_POOL_NAME);
    long heapSizeBytes = JvmInfo.jvmInfo().getMem().getHeapMax().getBytes();
    serializeRCFBufferPool = AccessController.doPrivileged(new PrivilegedAction<GenericObjectPool<LinkedBuffer>>() {

        @Override
        public GenericObjectPool<LinkedBuffer> run() {
            return new GenericObjectPool<>(new BasePooledObjectFactory<LinkedBuffer>() {

                @Override
                public LinkedBuffer create() throws Exception {
                    return LinkedBuffer.allocate(AnomalyDetectorSettings.SERIALIZATION_BUFFER_BYTES);
                }

                @Override
                public PooledObject<LinkedBuffer> wrap(LinkedBuffer obj) {
                    return new DefaultPooledObject<>(obj);
                }
            });
        }
    });
    serializeRCFBufferPool.setMaxTotal(AnomalyDetectorSettings.MAX_TOTAL_RCF_SERIALIZATION_BUFFERS);
    serializeRCFBufferPool.setMaxIdle(AnomalyDetectorSettings.MAX_TOTAL_RCF_SERIALIZATION_BUFFERS);
    serializeRCFBufferPool.setMinIdle(0);
    serializeRCFBufferPool.setBlockWhenExhausted(false);
    serializeRCFBufferPool.setTimeBetweenEvictionRuns(AnomalyDetectorSettings.HOURLY_MAINTENANCE);
    CheckpointDao checkpoint = new CheckpointDao(client, clientUtil, CommonName.CHECKPOINT_INDEX_NAME, gson, mapper, converter, new ThresholdedRandomCutForestMapper(), AccessController.doPrivileged((PrivilegedAction<Schema<ThresholdedRandomCutForestState>>) () -> RuntimeSchema.getSchema(ThresholdedRandomCutForestState.class)), HybridThresholdingModel.class, anomalyDetectionIndices, AnomalyDetectorSettings.MAX_CHECKPOINT_BYTES, serializeRCFBufferPool, AnomalyDetectorSettings.SERIALIZATION_BUFFER_BYTES, 1 - AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE);
    Random random = new Random(42);
    CheckpointWriteWorker checkpointWriteQueue = new CheckpointWriteWorker(heapSizeBytes, AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_SIZE_IN_BYTES, AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_MAX_HEAP_PERCENT, clusterService, random, adCircuitBreakerService, threadPool, settings, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, getClock(), AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, AnomalyDetectorSettings.QUEUE_MAINTENANCE, checkpoint, CommonName.CHECKPOINT_INDEX_NAME, AnomalyDetectorSettings.HOURLY_MAINTENANCE, stateManager, AnomalyDetectorSettings.HOURLY_MAINTENANCE);
    EntityCache cache = new PriorityCache(checkpoint, AnomalyDetectorSettings.DEDICATED_CACHE_SIZE.get(settings), AnomalyDetectorSettings.CHECKPOINT_TTL, AnomalyDetectorSettings.MAX_INACTIVE_ENTITIES, memoryTracker, AnomalyDetectorSettings.NUM_TREES, getClock(), clusterService, AnomalyDetectorSettings.HOURLY_MAINTENANCE, threadPool, checkpointWriteQueue, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT);
    CacheProvider cacheProvider = new CacheProvider(cache);
    EntityColdStarter entityColdStarter = new EntityColdStarter(getClock(), threadPool, stateManager, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.TIME_DECAY, AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, interpolator, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, settings, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue, AnomalyDetectorSettings.MAX_COLD_START_ROUNDS);
    EntityColdStartWorker coldstartQueue = new EntityColdStartWorker(heapSizeBytes, AnomalyDetectorSettings.ENTITY_REQUEST_SIZE_IN_BYTES, AnomalyDetectorSettings.ENTITY_COLD_START_QUEUE_MAX_HEAP_PERCENT, clusterService, random, adCircuitBreakerService, threadPool, settings, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, getClock(), AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, AnomalyDetectorSettings.QUEUE_MAINTENANCE, entityColdStarter, AnomalyDetectorSettings.HOURLY_MAINTENANCE, stateManager);
    ModelManager modelManager = new ModelManager(checkpoint, getClock(), AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.TIME_DECAY, AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, AnomalyDetectorSettings.MIN_PREVIEW_SIZE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, entityColdStarter, featureManager, memoryTracker);
    MultiEntityResultHandler multiEntityResultHandler = new MultiEntityResultHandler(client, settings, threadPool, anomalyDetectionIndices, this.clientUtil, this.indexUtils, clusterService);
    ResultWriteWorker resultWriteQueue = new ResultWriteWorker(heapSizeBytes, AnomalyDetectorSettings.RESULT_WRITE_QUEUE_SIZE_IN_BYTES, AnomalyDetectorSettings.RESULT_WRITE_QUEUE_MAX_HEAP_PERCENT, clusterService, random, adCircuitBreakerService, threadPool, settings, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, getClock(), AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, AnomalyDetectorSettings.QUEUE_MAINTENANCE, multiEntityResultHandler, xContentRegistry, stateManager, AnomalyDetectorSettings.HOURLY_MAINTENANCE);
    CheckpointReadWorker checkpointReadQueue = new CheckpointReadWorker(heapSizeBytes, AnomalyDetectorSettings.ENTITY_FEATURE_REQUEST_SIZE_IN_BYTES, AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_MAX_HEAP_PERCENT, clusterService, random, adCircuitBreakerService, threadPool, settings, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, getClock(), AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, AnomalyDetectorSettings.QUEUE_MAINTENANCE, modelManager, checkpoint, coldstartQueue, resultWriteQueue, stateManager, anomalyDetectionIndices, cacheProvider, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue);
    ColdEntityWorker coldEntityQueue = new ColdEntityWorker(heapSizeBytes, AnomalyDetectorSettings.ENTITY_FEATURE_REQUEST_SIZE_IN_BYTES, AnomalyDetectorSettings.COLD_ENTITY_QUEUE_MAX_HEAP_PERCENT, clusterService, random, adCircuitBreakerService, threadPool, settings, AnomalyDetectorSettings.MAX_QUEUED_TASKS_RATIO, getClock(), AnomalyDetectorSettings.MEDIUM_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.LOW_SEGMENT_PRUNE_RATIO, AnomalyDetectorSettings.MAINTENANCE_FREQ_CONSTANT, checkpointReadQueue, AnomalyDetectorSettings.HOURLY_MAINTENANCE, stateManager);
    ADDataMigrator dataMigrator = new ADDataMigrator(client, clusterService, xContentRegistry, anomalyDetectionIndices);
    HashRing hashRing = new HashRing(nodeFilter, getClock(), settings, client, clusterService, dataMigrator, modelManager);
    anomalyDetectorRunner = new AnomalyDetectorRunner(modelManager, featureManager, AnomalyDetectorSettings.MAX_PREVIEW_RESULTS);
    Map<String, ADStat<?>> stats = ImmutableMap.<String, ADStat<?>>builder().put(StatNames.AD_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.AD_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.AD_HC_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.AD_HC_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.MODEL_INFORMATION.getName(), new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider, settings, clusterService))).put(StatNames.ANOMALY_DETECTORS_INDEX_STATUS.getName(), new ADStat<>(true, new IndexStatusSupplier(indexUtils, AnomalyDetector.ANOMALY_DETECTORS_INDEX))).put(StatNames.ANOMALY_RESULTS_INDEX_STATUS.getName(), new ADStat<>(true, new IndexStatusSupplier(indexUtils, CommonName.ANOMALY_RESULT_INDEX_ALIAS))).put(StatNames.MODELS_CHECKPOINT_INDEX_STATUS.getName(), new ADStat<>(true, new IndexStatusSupplier(indexUtils, CommonName.CHECKPOINT_INDEX_NAME))).put(StatNames.ANOMALY_DETECTION_JOB_INDEX_STATUS.getName(), new ADStat<>(true, new IndexStatusSupplier(indexUtils, AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX))).put(StatNames.ANOMALY_DETECTION_STATE_STATUS.getName(), new ADStat<>(true, new IndexStatusSupplier(indexUtils, CommonName.DETECTION_STATE_INDEX))).put(StatNames.DETECTOR_COUNT.getName(), new ADStat<>(true, new SettableSupplier())).put(StatNames.SINGLE_ENTITY_DETECTOR_COUNT.getName(), new ADStat<>(true, new SettableSupplier())).put(StatNames.MULTI_ENTITY_DETECTOR_COUNT.getName(), new ADStat<>(true, new SettableSupplier())).put(StatNames.AD_EXECUTING_BATCH_TASK_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.AD_CANCELED_BATCH_TASK_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.AD_TOTAL_BATCH_TASK_EXECUTION_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.AD_BATCH_TASK_FAILURE_COUNT.getName(), new ADStat<>(false, new CounterSupplier())).put(StatNames.MODEL_COUNT.getName(), new ADStat<>(false, new ModelsOnNodeCountSupplier(modelManager, cacheProvider))).build();
    adStats = new ADStats(stats);
    adTaskCacheManager = new ADTaskCacheManager(settings, clusterService, memoryTracker);
    adTaskManager = new ADTaskManager(settings, clusterService, client, xContentRegistry, anomalyDetectionIndices, nodeFilter, hashRing, adTaskCacheManager, threadPool);
    AnomalyResultBulkIndexHandler anomalyResultBulkIndexHandler = new AnomalyResultBulkIndexHandler(client, settings, threadPool, this.clientUtil, this.indexUtils, clusterService, anomalyDetectionIndices);
    adBatchTaskRunner = new ADBatchTaskRunner(settings, threadPool, clusterService, client, adCircuitBreakerService, featureManager, adTaskManager, anomalyDetectionIndices, adStats, anomalyResultBulkIndexHandler, adTaskCacheManager, searchFeatureDao, hashRing, modelManager);
    ADSearchHandler adSearchHandler = new ADSearchHandler(settings, clusterService, client);
    // transport action handler constructors
    return ImmutableList.of(anomalyDetectionIndices, anomalyDetectorRunner, searchFeatureDao, singleFeatureLinearUniformInterpolator, interpolator, gson, jvmService, hashRing, featureManager, modelManager, stateManager, new ADClusterEventListener(clusterService, hashRing), adCircuitBreakerService, adStats, new MasterEventListener(clusterService, threadPool, client, getClock(), clientUtil, nodeFilter), nodeFilter, multiEntityResultHandler, checkpoint, cacheProvider, adTaskManager, adBatchTaskRunner, adSearchHandler, coldstartQueue, resultWriteQueue, checkpointReadQueue, checkpointWriteQueue, coldEntityQueue, entityColdStarter, adTaskCacheManager);
}

Example 5 with FeatureManager

use of org.opensearch.ad.feature.FeatureManager in project anomaly-detection by opensearch-project.

the class ProfileTransportActionTests method setUp.

@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    modelManager = mock(ModelManager.class);
    featureManager = mock(FeatureManager.class);
    when(featureManager.getShingleSize(any(String.class))).thenReturn(shingleSize);
    EntityCache cache = mock(EntityCache.class);
    cacheProvider = mock(CacheProvider.class);
    when(cacheProvider.get()).thenReturn(cache);
    when(cache.getActiveEntities(anyString())).thenReturn(activeEntities);
    when(cache.getTotalUpdates(anyString())).thenReturn(totalUpdates);
    Map<String, Long> multiEntityModelSizeMap = new HashMap<>();
    String modelId1 = "T4c3dXUBj-2IZN7itix__entity_app_3";
    String modelId2 = "T4c3dXUBj-2IZN7itix__entity_app_2";
    multiEntityModelSizeMap.put(modelId1, multiEntityModelSize);
    multiEntityModelSizeMap.put(modelId2, multiEntityModelSize);
    when(cache.getModelSize(anyString())).thenReturn(multiEntityModelSizeMap);
    List<ModelProfile> modelProfiles = new ArrayList<>();
    String field = "field";
    String fieldVal1 = "value1";
    String fieldVal2 = "value2";
    Entity entity1 = Entity.createSingleAttributeEntity(field, fieldVal1);
    Entity entity2 = Entity.createSingleAttributeEntity(field, fieldVal2);
    modelProfiles.add(new ModelProfile(modelId1, entity1, multiEntityModelSize));
    modelProfiles.add(new ModelProfile(modelId1, entity2, multiEntityModelSize));
    when(cache.getAllModelProfile(anyString())).thenReturn(modelProfiles);
    Map<String, Long> modelSizes = new HashMap<>();
    modelSizes.put(modelId, modelSize);
    when(modelManager.getModelSize(any(String.class))).thenReturn(modelSizes);
    Settings settings = Settings.builder().put("plugins.anomaly_detection.max_model_size_per_node", 100).build();
    action = new ProfileTransportAction(client().threadPool(), clusterService(), mock(TransportService.class), mock(ActionFilters.class), modelManager, featureManager, cacheProvider, settings);
    profilesToRetrieve = new HashSet<DetectorProfileName>();
    profilesToRetrieve.add(DetectorProfileName.COORDINATING_NODE);
}