Search in sources :

Example 11 with AnomalyDetectionException

use of org.opensearch.ad.common.exception.AnomalyDetectionException in project anomaly-detection by opensearch-project.

the class EntityColdStarterTests method testColdStartException.

public void testColdStartException() throws InterruptedException {
    Queue<double[]> samples = MLUtil.createQueueSamples(1);
    EntityModel model = new EntityModel(entity, samples, null);
    modelState = new ModelState<>(model, modelId, detectorId, ModelType.ENTITY.getName(), clock, priority);
    doAnswer(invocation -> {
        ActionListener<Optional<Long>> listener = invocation.getArgument(2);
        listener.onFailure(new AnomalyDetectionException(detectorId, ""));
        return null;
    }).when(searchFeatureDao).getEntityMinDataTime(any(), any(), any());
    entityColdStarter.trainModel(entity, detectorId, modelState, listener);
    assertTrue(stateManager.getLastDetectionError(detectorId) != null);
    checkSemaphoreRelease();
}
Also used : AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) Optional(java.util.Optional)

Example 12 with AnomalyDetectionException

use of org.opensearch.ad.common.exception.AnomalyDetectionException in project anomaly-detection by opensearch-project.

the class ADTaskManager method parseEntityFromString.

/**
 * Parse entity string value into Entity {@link Entity} instance.
 * @param entityValue entity value
 * @param adTask AD task
 * @return Entity instance
 */
public Entity parseEntityFromString(String entityValue, ADTask adTask) {
    AnomalyDetector detector = adTask.getDetector();
    if (detector.isMultiCategoryDetector()) {
        try {
            XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, entityValue);
            ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser);
            return Entity.parse(parser);
        } catch (IOException e) {
            String error = "Failed to parse string into entity";
            logger.debug(error, e);
            throw new AnomalyDetectionException(error);
        }
    } else if (detector.isMultientityDetector()) {
        return Entity.createSingleAttributeEntity(detector.getCategoryField().get(0), entityValue);
    }
    throw new IllegalArgumentException("Fail to parse to Entity for single flow detector");
}
Also used : AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) IOException(java.io.IOException) ADTaskType.taskTypeToString(org.opensearch.ad.model.ADTaskType.taskTypeToString) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) XContentParser(org.opensearch.common.xcontent.XContentParser)

Example 13 with AnomalyDetectionException

use of org.opensearch.ad.common.exception.AnomalyDetectionException in project anomaly-detection by opensearch-project.

the class ADBatchTaskRunner method getDateRangeOfSourceData.

private void getDateRangeOfSourceData(ADTask adTask, BiConsumer<Long, Long> consumer, ActionListener<String> internalListener) {
    String taskId = adTask.getTaskId();
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().aggregation(AggregationBuilders.min(AGG_NAME_MIN_TIME).field(adTask.getDetector().getTimeField())).aggregation(AggregationBuilders.max(AGG_NAME_MAX_TIME).field(adTask.getDetector().getTimeField())).size(0);
    if (adTask.getEntity() != null && adTask.getEntity().getAttributes().size() > 0) {
        BoolQueryBuilder query = new BoolQueryBuilder();
        adTask.getEntity().getAttributes().entrySet().forEach(entity -> query.filter(new TermQueryBuilder(entity.getKey(), entity.getValue())));
        searchSourceBuilder.query(query);
    }
    SearchRequest request = new SearchRequest().indices(adTask.getDetector().getIndices().toArray(new String[0])).source(searchSourceBuilder);
    client.search(request, ActionListener.wrap(r -> {
        InternalMin minAgg = r.getAggregations().get(AGG_NAME_MIN_TIME);
        InternalMax maxAgg = r.getAggregations().get(AGG_NAME_MAX_TIME);
        double minValue = minAgg.getValue();
        double maxValue = maxAgg.getValue();
        // If time field not exist or there is no value, will return infinity value
        if (minValue == Double.POSITIVE_INFINITY) {
            internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the time field"));
            return;
        }
        long interval = ((IntervalTimeConfiguration) adTask.getDetector().getDetectionInterval()).toDuration().toMillis();
        DetectionDateRange detectionDateRange = adTask.getDetectionDateRange();
        long dataStartTime = detectionDateRange.getStartTime().toEpochMilli();
        long dataEndTime = detectionDateRange.getEndTime().toEpochMilli();
        long minDate = (long) minValue;
        long maxDate = (long) maxValue;
        if (minDate >= dataEndTime || maxDate <= dataStartTime) {
            internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the detection date range"));
            return;
        }
        if (minDate > dataStartTime) {
            dataStartTime = minDate;
        }
        if (maxDate < dataEndTime) {
            dataEndTime = maxDate;
        }
        // normalize start/end time to make it consistent with feature data agg result
        dataStartTime = dataStartTime - dataStartTime % interval;
        dataEndTime = dataEndTime - dataEndTime % interval;
        logger.debug("adjusted date range: start: {}, end: {}, taskId: {}", dataStartTime, dataEndTime, taskId);
        if ((dataEndTime - dataStartTime) < NUM_MIN_SAMPLES * interval) {
            internalListener.onFailure(new AnomalyDetectionException("There is not enough data to train model").countedInStats(false));
            return;
        }
        consumer.accept(dataStartTime, dataEndTime);
    }, e -> {
        internalListener.onFailure(e);
    }));
}
Also used : AnomalyResultBulkIndexHandler(org.opensearch.ad.transport.handler.AnomalyResultBulkIndexHandler) ModelManager(org.opensearch.ad.ml.ModelManager) HashRing(org.opensearch.ad.cluster.HashRing) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) ADIndex(org.opensearch.ad.indices.ADIndex) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) BATCH_TASK_PIECE_SIZE(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE) ADStatsNodeResponse(org.opensearch.ad.transport.ADStatsNodeResponse) MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS) Client(org.opensearch.client.Client) AD_BATCH_TASK_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_BATCH_TASK_THREAD_POOL_NAME) TimeValue(org.opensearch.common.unit.TimeValue) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Settings(org.opensearch.common.settings.Settings) NO_ELIGIBLE_NODE_TO_RUN_DETECTOR(org.opensearch.ad.constant.CommonErrorMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR) TransportService(org.opensearch.transport.TransportService) RandomCutForest(com.amazon.randomcutforest.RandomCutForest) Logger(org.apache.logging.log4j.Logger) PriorityTracker(org.opensearch.ad.caching.PriorityTracker) ExceptionUtil(org.opensearch.ad.util.ExceptionUtil) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) SearchFeatureDao(org.opensearch.ad.feature.SearchFeatureDao) CheckedRunnable(org.opensearch.common.CheckedRunnable) EXECUTION_END_TIME_FIELD(org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD) InjectSecurity(org.opensearch.commons.InjectSecurity) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) BATCH_TASK_PIECE_INTERVAL_SECONDS(org.opensearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS) ADCircuitBreakerService(org.opensearch.ad.breaker.ADCircuitBreakerService) MAX_BATCH_TASK_PER_NODE(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE) StringTerms(org.opensearch.search.aggregations.bucket.terms.StringTerms) ThreadPool(org.opensearch.threadpool.ThreadPool) EnabledSetting(org.opensearch.ad.settings.EnabledSetting) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) BiConsumer(java.util.function.BiConsumer) SearchRequest(org.opensearch.action.search.SearchRequest) INIT_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD) SinglePointFeatures(org.opensearch.ad.feature.SinglePointFeatures) FeatureManager(org.opensearch.ad.feature.FeatureManager) MAX_TOP_ENTITIES_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TOP_ENTITIES_FOR_HISTORICAL_ANALYSIS) ADStatsNodesAction(org.opensearch.ad.transport.ADStatsNodesAction) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) ClusterService(org.opensearch.cluster.service.ClusterService) StatNames(org.opensearch.ad.stats.StatNames) ParseUtils(org.opensearch.ad.util.ParseUtils) InternalMin(org.opensearch.search.aggregations.metrics.InternalMin) AD_EXECUTING_BATCH_TASK_COUNT(org.opensearch.ad.stats.StatNames.AD_EXECUTING_BATCH_TASK_COUNT) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADStats(org.opensearch.ad.stats.ADStats) ParseUtils.isNullOrEmpty(org.opensearch.ad.util.ParseUtils.isNullOrEmpty) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) ADStatsRequest(org.opensearch.ad.transport.ADStatsRequest) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) AggregationBuilder(org.opensearch.search.aggregations.AggregationBuilder) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) TASK_PROGRESS_FIELD(org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) ADBatchTaskRemoteExecutionAction(org.opensearch.ad.transport.ADBatchTaskRemoteExecutionAction) ADTaskType(org.opensearch.ad.model.ADTaskType) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) AnomalyResult(org.opensearch.ad.model.AnomalyResult) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) DEFAULT_JVM_HEAP_USAGE_THRESHOLD(org.opensearch.ad.breaker.MemoryCircuitBreaker.DEFAULT_JVM_HEAP_USAGE_THRESHOLD) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) NUM_MIN_SAMPLES(org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES) Optional(java.util.Optional) AnomalyDescriptor(com.amazon.randomcutforest.parkservices.AnomalyDescriptor) ADTask(org.opensearch.ad.model.ADTask) FeatureData(org.opensearch.ad.model.FeatureData) HashMap(java.util.HashMap) Deque(java.util.Deque) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) CURRENT_PIECE_FIELD(org.opensearch.ad.model.ADTask.CURRENT_PIECE_FIELD) ImmutableList(com.google.common.collect.ImmutableList) ADBatchAnomalyResultResponse(org.opensearch.ad.transport.ADBatchAnomalyResultResponse) JVM_HEAP_USAGE(org.opensearch.ad.stats.InternalStatNames.JVM_HEAP_USAGE) EndRunException(org.opensearch.ad.common.exception.EndRunException) ADBatchAnomalyResultRequest(org.opensearch.ad.transport.ADBatchAnomalyResultRequest) AGG_NAME_MAX_TIME(org.opensearch.ad.constant.CommonName.AGG_NAME_MAX_TIME) RangeQueryBuilder(org.opensearch.index.query.RangeQueryBuilder) MAX_TOP_ENTITIES_LIMIT_FOR_HISTORICAL_ANALYSIS(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_TOP_ENTITIES_LIMIT_FOR_HISTORICAL_ANALYSIS) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) STATE_FIELD(org.opensearch.ad.model.ADTask.STATE_FIELD) WORKER_NODE_FIELD(org.opensearch.ad.model.ADTask.WORKER_NODE_FIELD) Entity(org.opensearch.ad.model.Entity) TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) ThresholdedRandomCutForest(com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest) Clock(java.time.Clock) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) LogManager(org.apache.logging.log4j.LogManager) AGG_NAME_MIN_TIME(org.opensearch.ad.constant.CommonName.AGG_NAME_MIN_TIME) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) SearchRequest(org.opensearch.action.search.SearchRequest) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) BoolQueryBuilder(org.opensearch.index.query.BoolQueryBuilder) InternalMin(org.opensearch.search.aggregations.metrics.InternalMin) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) TermQueryBuilder(org.opensearch.index.query.TermQueryBuilder) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) DetectionDateRange(org.opensearch.ad.model.DetectionDateRange) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder)

Example 14 with AnomalyDetectionException

use of org.opensearch.ad.common.exception.AnomalyDetectionException in project anomaly-detection by opensearch-project.

the class RCFPollingTransportAction method doExecute.

@Override
protected void doExecute(Task task, RCFPollingRequest request, ActionListener<RCFPollingResponse> listener) {
    String adID = request.getAdID();
    String rcfModelID = SingleStreamModelIdMapper.getRcfModelId(adID, 0);
    Optional<DiscoveryNode> rcfNode = hashRing.getOwningNodeWithSameLocalAdVersionForRealtimeAD(rcfModelID);
    if (!rcfNode.isPresent()) {
        listener.onFailure(new AnomalyDetectionException(adID, NO_NODE_FOUND_MSG));
        return;
    }
    String rcfNodeId = rcfNode.get().getId();
    DiscoveryNode localNode = clusterService.localNode();
    if (localNode.getId().equals(rcfNodeId)) {
        modelManager.getTotalUpdates(rcfModelID, adID, ActionListener.wrap(totalUpdates -> listener.onResponse(new RCFPollingResponse(totalUpdates)), e -> listener.onFailure(new AnomalyDetectionException(adID, FAIL_TO_GET_RCF_UPDATE_MSG, e))));
    } else if (request.remoteAddress() == null) {
        // redirect if request comes from local host.
        // If a request comes from remote machine, it is already redirected.
        // One redirection should be enough.
        // We don't want a potential infinite loop due to any bug and thus give up.
        LOG.info("Sending RCF polling request to {} for model {}", rcfNodeId, rcfModelID);
        try {
            transportService.sendRequest(rcfNode.get(), RCFPollingAction.NAME, request, option, new TransportResponseHandler<RCFPollingResponse>() {

                @Override
                public RCFPollingResponse read(StreamInput in) throws IOException {
                    return new RCFPollingResponse(in);
                }

                @Override
                public void handleResponse(RCFPollingResponse response) {
                    listener.onResponse(response);
                }

                @Override
                public void handleException(TransportException exp) {
                    listener.onFailure(exp);
                }

                @Override
                public String executor() {
                    return ThreadPool.Names.SAME;
                }
            });
        } catch (Exception e) {
            LOG.error(String.format(Locale.ROOT, "Fail to poll RCF models for {}", adID), e);
            listener.onFailure(new AnomalyDetectionException(adID, FAIL_TO_GET_RCF_UPDATE_MSG, e));
        }
    } else {
        LOG.error("Fail to poll rcf for model {} due to an unexpected bug.", rcfModelID);
        listener.onFailure(new AnomalyDetectionException(adID, NO_NODE_FOUND_MSG));
    }
}
Also used : AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) StreamInput(org.opensearch.common.io.stream.StreamInput) HandledTransportAction(org.opensearch.action.support.HandledTransportAction) ModelManager(org.opensearch.ad.ml.ModelManager) HashRing(org.opensearch.ad.cluster.HashRing) ThreadPool(org.opensearch.threadpool.ThreadPool) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) TransportResponseHandler(org.opensearch.transport.TransportResponseHandler) IOException(java.io.IOException) Settings(org.opensearch.common.settings.Settings) Task(org.opensearch.tasks.Task) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) TransportService(org.opensearch.transport.TransportService) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) ActionFilters(org.opensearch.action.support.ActionFilters) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Logger(org.apache.logging.log4j.Logger) Locale(java.util.Locale) ClusterService(org.opensearch.cluster.service.ClusterService) Optional(java.util.Optional) SingleStreamModelIdMapper(org.opensearch.ad.ml.SingleStreamModelIdMapper) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) LogManager(org.apache.logging.log4j.LogManager) TransportException(org.opensearch.transport.TransportException) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) TransportResponseHandler(org.opensearch.transport.TransportResponseHandler) StreamInput(org.opensearch.common.io.stream.StreamInput) TransportException(org.opensearch.transport.TransportException) IOException(java.io.IOException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) TransportException(org.opensearch.transport.TransportException)

Example 15 with AnomalyDetectionException

use of org.opensearch.ad.common.exception.AnomalyDetectionException in project anomaly-detection by opensearch-project.

the class AnomalyResultTransportAction method coldStart.

private void coldStart(AnomalyDetector detector) {
    String detectorId = detector.getDetectorId();
    // If last cold start is not finished, we don't trigger another one
    if (stateManager.isColdStartRunning(detectorId)) {
        return;
    }
    final Releasable coldStartFinishingCallback = stateManager.markColdStartRunning(detectorId);
    ActionListener<Optional<double[][]>> listener = ActionListener.wrap(trainingData -> {
        if (trainingData.isPresent()) {
            double[][] dataPoints = trainingData.get();
            ActionListener<Void> trainModelListener = ActionListener.wrap(res -> {
                LOG.info("Succeeded in training {}", detectorId);
            }, exception -> {
                if (exception instanceof AnomalyDetectionException) {
                    // e.g., partitioned model exceeds memory limit
                    stateManager.setException(detectorId, exception);
                } else if (exception instanceof IllegalArgumentException) {
                    // IllegalArgumentException due to invalid training data
                    stateManager.setException(detectorId, new EndRunException(detectorId, "Invalid training data", exception, false));
                } else if (exception instanceof OpenSearchTimeoutException) {
                    stateManager.setException(detectorId, new InternalFailure(detectorId, "Time out while indexing cold start checkpoint", exception));
                } else {
                    stateManager.setException(detectorId, new EndRunException(detectorId, "Error while training model", exception, false));
                }
            });
            modelManager.trainModel(detector, dataPoints, new ThreadedActionListener<>(LOG, threadPool, AnomalyDetectorPlugin.AD_THREAD_POOL_NAME, trainModelListener, false));
        } else {
            stateManager.setException(detectorId, new EndRunException(detectorId, "Cannot get training data", false));
        }
    }, exception -> {
        if (exception instanceof OpenSearchTimeoutException) {
            stateManager.setException(detectorId, new InternalFailure(detectorId, "Time out while getting training data", exception));
        } else if (exception instanceof AnomalyDetectionException) {
            // e.g., Invalid search query
            stateManager.setException(detectorId, exception);
        } else {
            stateManager.setException(detectorId, new EndRunException(detectorId, "Error while cold start", exception, false));
        }
    });
    final ActionListener<Optional<double[][]>> listenerWithReleaseCallback = ActionListener.runAfter(listener, coldStartFinishingCallback::close);
    threadPool.executor(AnomalyDetectorPlugin.AD_THREAD_POOL_NAME).execute(() -> featureManager.getColdStartData(detector, new ThreadedActionListener<>(LOG, threadPool, AnomalyDetectorPlugin.AD_THREAD_POOL_NAME, listenerWithReleaseCallback, false)));
}
Also used : AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) EndRunException(org.opensearch.ad.common.exception.EndRunException) OpenSearchTimeoutException(org.opensearch.OpenSearchTimeoutException) Optional(java.util.Optional) InternalFailure(org.opensearch.ad.common.exception.InternalFailure) Releasable(org.opensearch.common.lease.Releasable)

Aggregations

AnomalyDetectionException (org.opensearch.ad.common.exception.AnomalyDetectionException)28 ActionListener (org.opensearch.action.ActionListener)10 IOException (java.io.IOException)9 Optional (java.util.Optional)8 EndRunException (org.opensearch.ad.common.exception.EndRunException)7 AnomalyDetector (org.opensearch.ad.model.AnomalyDetector)7 LogManager (org.apache.logging.log4j.LogManager)6 Logger (org.apache.logging.log4j.Logger)6 DiscoveryNode (org.opensearch.cluster.node.DiscoveryNode)6 ClusterService (org.opensearch.cluster.service.ClusterService)6 Settings (org.opensearch.common.settings.Settings)6 ArrayList (java.util.ArrayList)5 List (java.util.List)5 ModelManager (org.opensearch.ad.ml.ModelManager)5 Instant (java.time.Instant)4 Map (java.util.Map)4 ThreadedActionListener (org.opensearch.action.support.ThreadedActionListener)4 LimitExceededException (org.opensearch.ad.common.exception.LimitExceededException)4 ResourceNotFoundException (org.opensearch.ad.common.exception.ResourceNotFoundException)4 Entity (org.opensearch.ad.model.Entity)4