Search in sources :

Example 1 with DetectorProfileName

use of org.opensearch.ad.model.DetectorProfileName in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunner method updateRealtimeTask.

private void updateRealtimeTask(AnomalyResultResponse response, String detectorId) {
    if (response.isHCDetector() != null && response.isHCDetector() && !adTaskManager.skipUpdateHCRealtimeTask(detectorId, response.getError())) {
        DiscoveryNode[] dataNodes = nodeFilter.getEligibleDataNodes();
        Set<DetectorProfileName> profiles = new HashSet<>();
        profiles.add(DetectorProfileName.INIT_PROGRESS);
        ProfileRequest profileRequest = new ProfileRequest(detectorId, profiles, true, dataNodes);
        client.execute(ProfileAction.INSTANCE, profileRequest, ActionListener.wrap(r -> {
            log.debug("Update latest realtime task for HC detector {}, total updates: {}", detectorId, r.getTotalUpdates());
            updateLatestRealtimeTask(detectorId, null, r.getTotalUpdates(), response.getDetectorIntervalInMinutes(), response.getError());
        }, e -> {
            log.error("Failed to update latest realtime task for " + detectorId, e);
        }));
    } else {
        log.debug("Update latest realtime task for SINGLE detector {}, total updates: {}", detectorId, response.getRcfTotalUpdates());
        updateLatestRealtimeTask(detectorId, null, response.getRcfTotalUpdates(), response.getDetectorIntervalInMinutes(), response.getError());
    }
}
Also used : ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) XContentParser(org.opensearch.common.xcontent.XContentParser) AD_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_THREAD_POOL_NAME) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) WriteRequest(org.opensearch.action.support.WriteRequest) ADIndex(org.opensearch.ad.indices.ADIndex) ActionListener(org.opensearch.action.ActionListener) GetResponse(org.opensearch.action.get.GetResponse) Client(org.opensearch.client.Client) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) Set(java.util.Set) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskManager(org.opensearch.ad.task.ADTaskManager) Settings(org.opensearch.common.settings.Settings) Instant(java.time.Instant) UPDATED(org.opensearch.action.DocWriteResponse.Result.UPDATED) List(java.util.List) AnomalyResultAction(org.opensearch.ad.transport.AnomalyResultAction) Logger(org.apache.logging.log4j.Logger) AnomalyResult(org.opensearch.ad.model.AnomalyResult) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) XContentType(org.opensearch.common.xcontent.XContentType) InjectSecurity(org.opensearch.commons.InjectSecurity) JobExecutionContext(org.opensearch.jobscheduler.spi.JobExecutionContext) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) AnomalyIndexHandler(org.opensearch.ad.transport.handler.AnomalyIndexHandler) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) FeatureData(org.opensearch.ad.model.FeatureData) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) ProfileRequest(org.opensearch.ad.transport.ProfileRequest) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) LockModel(org.opensearch.jobscheduler.spi.LockModel) ExecutorService(java.util.concurrent.ExecutorService) EndRunException(org.opensearch.ad.common.exception.EndRunException) LockService(org.opensearch.jobscheduler.spi.utils.LockService) InternalFailure(org.opensearch.ad.common.exception.InternalFailure) GetRequest(org.opensearch.action.get.GetRequest) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) ScheduledJobParameter(org.opensearch.jobscheduler.spi.ScheduledJobParameter) AnomalyResultTransportAction(org.opensearch.ad.transport.AnomalyResultTransportAction) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) User(org.opensearch.commons.authuser.User) AnomalyResultResponse(org.opensearch.ad.transport.AnomalyResultResponse) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ScheduledJobRunner(org.opensearch.jobscheduler.spi.ScheduledJobRunner) IndexRequest(org.opensearch.action.index.IndexRequest) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) LogManager(org.apache.logging.log4j.LogManager) ProfileAction(org.opensearch.ad.transport.ProfileAction) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) ProfileRequest(org.opensearch.ad.transport.ProfileRequest) HashSet(java.util.HashSet)

Example 2 with DetectorProfileName

use of org.opensearch.ad.model.DetectorProfileName in project anomaly-detection by opensearch-project.

the class MultiEntityProfileRunnerTests method setUp.

@SuppressWarnings("unchecked")
@Before
@Override
public void setUp() throws Exception {
    super.setUp();
    client = mock(Client.class);
    nodeFilter = mock(DiscoveryNodeFilterer.class);
    requiredSamples = 128;
    detectorId = "A69pa3UBHuCbh-emo9oR";
    detector = TestHelpers.randomAnomalyDetectorUsingCategoryFields(detectorId, Arrays.asList("a"));
    result = new DetectorInternalState.Builder().lastUpdateTime(Instant.now());
    job = TestHelpers.randomAnomalyDetectorJob(true);
    adTaskManager = mock(ADTaskManager.class);
    transportService = mock(TransportService.class);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        Consumer<Optional<ADTask>> function = (Consumer<Optional<ADTask>>) args[2];
        function.accept(Optional.of(TestHelpers.randomAdTask()));
        return null;
    }).when(adTaskManager).getAndExecuteOnLatestDetectorLevelTask(any(), any(), any(), any(), anyBoolean(), any());
    runner = new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, requiredSamples, transportService, adTaskManager);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        GetRequest request = (GetRequest) args[0];
        ActionListener<GetResponse> listener = (ActionListener<GetResponse>) args[1];
        String indexName = request.index();
        if (indexName.equals(ANOMALY_DETECTORS_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(detector, detector.getDetectorId(), AnomalyDetector.ANOMALY_DETECTORS_INDEX));
        } else if (indexName.equals(CommonName.DETECTION_STATE_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(result.build(), detector.getDetectorId(), CommonName.DETECTION_STATE_INDEX));
        } else if (indexName.equals(ANOMALY_DETECTOR_JOB_INDEX)) {
            listener.onResponse(TestHelpers.createGetResponse(job, detector.getDetectorId(), AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX));
        }
        return null;
    }).when(client).get(any(), any());
    stateNError = new HashSet<DetectorProfileName>();
    stateNError.add(DetectorProfileName.ERROR);
    stateNError.add(DetectorProfileName.STATE);
}
Also used : DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) Optional(java.util.Optional) GetResponse(org.opensearch.action.get.GetResponse) Consumer(java.util.function.Consumer) ActionListener(org.opensearch.action.ActionListener) ADTaskManager(org.opensearch.ad.task.ADTaskManager) TransportService(org.opensearch.transport.TransportService) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) GetRequest(org.opensearch.action.get.GetRequest) ADTask(org.opensearch.ad.model.ADTask) Client(org.opensearch.client.Client) Before(org.junit.Before)

Example 3 with DetectorProfileName

use of org.opensearch.ad.model.DetectorProfileName in project anomaly-detection by opensearch-project.

the class AbstractProfileRunnerTests method setUpOnce.

@BeforeClass
public static void setUpOnce() {
    stateOnly = new HashSet<DetectorProfileName>();
    stateOnly.add(DetectorProfileName.STATE);
    stateNError = new HashSet<DetectorProfileName>();
    stateNError.add(DetectorProfileName.ERROR);
    stateNError.add(DetectorProfileName.STATE);
    stateInitProgress = new HashSet<DetectorProfileName>();
    stateInitProgress.add(DetectorProfileName.INIT_PROGRESS);
    stateInitProgress.add(DetectorProfileName.STATE);
    modelProfile = new HashSet<DetectorProfileName>(Arrays.asList(DetectorProfileName.SHINGLE_SIZE, DetectorProfileName.MODELS, DetectorProfileName.COORDINATING_NODE, DetectorProfileName.TOTAL_SIZE_IN_BYTES));
    totalInitProgress = new HashSet<DetectorProfileName>(Arrays.asList(DetectorProfileName.TOTAL_ENTITIES, DetectorProfileName.INIT_PROGRESS));
    initProgressErrorProfile = new HashSet<DetectorProfileName>(Arrays.asList(DetectorProfileName.INIT_PROGRESS, DetectorProfileName.ERROR));
    clusterName = "test-cluster-name";
    discoveryNode1 = new DiscoveryNode("nodeName1", "node1", new TransportAddress(TransportAddress.META_ADDRESS, 9300), emptyMap(), emptySet(), Version.CURRENT);
}
Also used : DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) TransportAddress(org.opensearch.common.transport.TransportAddress) BeforeClass(org.junit.BeforeClass)

Example 4 with DetectorProfileName

use of org.opensearch.ad.model.DetectorProfileName in project anomaly-detection by opensearch-project.

the class ProfileTests method testProfileNodeRequest.

@Test
public void testProfileNodeRequest() throws IOException {
    Set<DetectorProfileName> profilesToRetrieve = new HashSet<DetectorProfileName>();
    profilesToRetrieve.add(DetectorProfileName.COORDINATING_NODE);
    ProfileRequest ProfileRequest = new ProfileRequest(detectorId, profilesToRetrieve, false);
    ProfileNodeRequest ProfileNodeRequest = new ProfileNodeRequest(ProfileRequest);
    assertEquals("ProfileNodeRequest has the wrong detector id", ProfileNodeRequest.getDetectorId(), detectorId);
    assertEquals("ProfileNodeRequest has the wrong ProfileRequest", ProfileNodeRequest.getProfilesToBeRetrieved(), profilesToRetrieve);
    // Test serialization
    BytesStreamOutput output = new BytesStreamOutput();
    ProfileNodeRequest.writeTo(output);
    StreamInput streamInput = output.bytes().streamInput();
    ProfileNodeRequest nodeRequest = new ProfileNodeRequest(streamInput);
    assertEquals("serialization has the wrong detector id", nodeRequest.getDetectorId(), detectorId);
    assertEquals("serialization has the wrong ProfileRequest", nodeRequest.getProfilesToBeRetrieved(), profilesToRetrieve);
}
Also used : DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) StreamInput(org.opensearch.common.io.stream.StreamInput) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 5 with DetectorProfileName

use of org.opensearch.ad.model.DetectorProfileName in project anomaly-detection by opensearch-project.

the class ProfileTests method testProfileRequest.

@Test
public void testProfileRequest() throws IOException {
    String detectorId = "123";
    Set<DetectorProfileName> profilesToRetrieve = new HashSet<DetectorProfileName>();
    profilesToRetrieve.add(DetectorProfileName.COORDINATING_NODE);
    ProfileRequest profileRequest = new ProfileRequest(detectorId, profilesToRetrieve, false);
    // Test Serialization
    BytesStreamOutput output = new BytesStreamOutput();
    profileRequest.writeTo(output);
    StreamInput streamInput = output.bytes().streamInput();
    ProfileRequest readRequest = new ProfileRequest(streamInput);
    assertEquals("Serialization has the wrong profiles to be retrieved", readRequest.getProfilesToBeRetrieved(), profileRequest.getProfilesToBeRetrieved());
    assertEquals("Serialization has the wrong detector id", readRequest.getDetectorId(), profileRequest.getDetectorId());
}
Also used : DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) StreamInput(org.opensearch.common.io.stream.StreamInput) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

DetectorProfileName (org.opensearch.ad.model.DetectorProfileName)14 DiscoveryNode (org.opensearch.cluster.node.DiscoveryNode)8 ActionListener (org.opensearch.action.ActionListener)6 List (java.util.List)5 Set (java.util.Set)5 GetRequest (org.opensearch.action.get.GetRequest)5 AnomalyDetectorSettings (org.opensearch.ad.settings.AnomalyDetectorSettings)5 ADTaskManager (org.opensearch.ad.task.ADTaskManager)5 DiscoveryNodeFilterer (org.opensearch.ad.util.DiscoveryNodeFilterer)5 Client (org.opensearch.client.Client)5 ArrayList (java.util.ArrayList)4 Map (java.util.Map)4 LogManager (org.apache.logging.log4j.LogManager)4 Logger (org.apache.logging.log4j.Logger)4 ResourceNotFoundException (org.opensearch.ad.common.exception.ResourceNotFoundException)4 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)4 ANOMALY_DETECTORS_INDEX (org.opensearch.ad.model.AnomalyDetector.ANOMALY_DETECTORS_INDEX)4 AnomalyDetectorJob (org.opensearch.ad.model.AnomalyDetectorJob)4 ANOMALY_DETECTOR_JOB_INDEX (org.opensearch.ad.model.AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX)4 DetectorProfile (org.opensearch.ad.model.DetectorProfile)4