Search in sources :

Example 1 with ClientException

use of org.opensearch.ad.common.exception.ClientException in project anomaly-detection by opensearch-project.

the class AnomalyResultTransportAction method doExecute.

/**
 * All the exceptions thrown by AD is a subclass of AnomalyDetectionException.
 *  ClientException is a subclass of AnomalyDetectionException. All exception visible to
 *   Client is under ClientVisible. Two classes directly extends ClientException:
 *   - InternalFailure for "root cause unknown failure. Maybe transient." We can continue the
 *    detector running.
 *   - EndRunException for "failures that might impact the customer." The method endNow() is
 *    added to indicate whether the client should immediately terminate running a detector.
 *      + endNow() returns true for "unrecoverable issue". We want to terminate the detector run
 *       immediately.
 *      + endNow() returns false for "maybe unrecoverable issue but worth retrying a few more
 *       times." We want to wait for a few more times on different requests before terminating
 *        the detector run.
 *
 *  AD may not be able to get an anomaly grade but can find a feature vector.  Consider the
 *   case when the shingle is not ready.  In that case, AD just put NaN as anomaly grade and
 *    return the feature vector. If AD cannot even find a feature vector, AD throws
 *     EndRunException if there is an issue or returns empty response (all the numeric fields
 *      are Double.NaN and feature array is empty.  Do so so that customer can write painless
 *       script.) otherwise.
 *
 *  Known causes of EndRunException with endNow returning false:
 *   + training data for cold start not available
 *   + cold start cannot succeed
 *   + unknown prediction error
 *   + memory circuit breaker tripped
 *   + invalid search query
 *
 *  Known causes of EndRunException with endNow returning true:
 *   + a model partition's memory size reached limit
 *   + models' total memory size reached limit
 *   + Having trouble querying feature data due to
 *    * index does not exist
 *    * all features have been disabled
 *
 *   + anomaly detector is not available
 *   + AD plugin is disabled
 *   + training data is invalid due to serious internal bug(s)
 *
 *  Known causes of InternalFailure:
 *   + threshold model node is not available
 *   + cluster read/write is blocked
 *   + cold start hasn't been finished
 *   + fail to get all of rcf model nodes' responses
 *   + fail to get threshold model node's response
 *   + RCF/Threshold model node failing to get checkpoint to restore model before timeout
 *   + Detection is throttle because previous detection query is running
 */
@Override
protected void doExecute(Task task, ActionRequest actionRequest, ActionListener<AnomalyResultResponse> listener) {
    try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
        AnomalyResultRequest request = AnomalyResultRequest.fromActionRequest(actionRequest);
        String adID = request.getAdID();
        ActionListener<AnomalyResultResponse> original = listener;
        listener = ActionListener.wrap(r -> {
            hcDetectors.remove(adID);
            original.onResponse(r);
        }, e -> {
            // we will not count it in failure stats.
            if (!(e instanceof AnomalyDetectionException) || ((AnomalyDetectionException) e).isCountedInStats()) {
                adStats.getStat(StatNames.AD_EXECUTE_FAIL_COUNT.getName()).increment();
                if (hcDetectors.contains(adID)) {
                    adStats.getStat(StatNames.AD_HC_EXECUTE_FAIL_COUNT.getName()).increment();
                }
            }
            hcDetectors.remove(adID);
            original.onFailure(e);
        });
        if (!EnabledSetting.isADPluginEnabled()) {
            throw new EndRunException(adID, CommonErrorMessages.DISABLED_ERR_MSG, true).countedInStats(false);
        }
        adStats.getStat(StatNames.AD_EXECUTE_REQUEST_COUNT.getName()).increment();
        if (adCircuitBreakerService.isOpen()) {
            listener.onFailure(new LimitExceededException(adID, CommonErrorMessages.MEMORY_CIRCUIT_BROKEN_ERR_MSG, false));
            return;
        }
        try {
            stateManager.getAnomalyDetector(adID, onGetDetector(listener, adID, request));
        } catch (Exception ex) {
            handleExecuteException(ex, listener, adID);
        }
    } catch (Exception e) {
        LOG.error(e);
        listener.onFailure(e);
    }
}
Also used : ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ModelManager(org.opensearch.ad.ml.ModelManager) HashRing(org.opensearch.ad.cluster.HashRing) ADStats(org.opensearch.ad.stats.ADStats) ActionRequest(org.opensearch.action.ActionRequest) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) Map(java.util.Map) NodeClosedException(org.opensearch.node.NodeClosedException) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) NodeNotConnectedException(org.opensearch.transport.NodeNotConnectedException) Client(org.opensearch.client.Client) ActionNotFoundTransportException(org.opensearch.transport.ActionNotFoundTransportException) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Set(java.util.Set) ExceptionsHelper(org.opensearch.ExceptionsHelper) ADTaskManager(org.opensearch.ad.task.ADTaskManager) Settings(org.opensearch.common.settings.Settings) Task(org.opensearch.tasks.Task) RestStatus(org.opensearch.rest.RestStatus) Collectors(java.util.stream.Collectors) TransportService(org.opensearch.transport.TransportService) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ExceptionUtil(org.opensearch.ad.util.ExceptionUtil) NodeStateManager(org.opensearch.ad.NodeStateManager) Entry(java.util.Map.Entry) OpenSearchTimeoutException(org.opensearch.OpenSearchTimeoutException) ActionListenerResponseHandler(org.opensearch.action.ActionListenerResponseHandler) Optional(java.util.Optional) SingleStreamModelIdMapper(org.opensearch.ad.ml.SingleStreamModelIdMapper) ConnectTransportException(org.opensearch.transport.ConnectTransportException) ReceiveTimeoutTransportException(org.opensearch.transport.ReceiveTimeoutTransportException) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) ADCircuitBreakerService(org.opensearch.ad.breaker.ADCircuitBreakerService) MAX_ENTITIES_PER_QUERY(org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_ENTITIES_PER_QUERY) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) HandledTransportAction(org.opensearch.action.support.HandledTransportAction) NotSerializableExceptionWrapper(org.opensearch.common.io.stream.NotSerializableExceptionWrapper) ThreadPool(org.opensearch.threadpool.ThreadPool) FeatureData(org.opensearch.ad.model.FeatureData) EnabledSetting(org.opensearch.ad.settings.EnabledSetting) Releasable(org.opensearch.common.lease.Releasable) IndicesOptions(org.opensearch.action.support.IndicesOptions) ThreadedActionListener(org.opensearch.action.support.ThreadedActionListener) AtomicReference(java.util.concurrent.atomic.AtomicReference) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) PAGE_SIZE(org.opensearch.ad.settings.AnomalyDetectorSettings.PAGE_SIZE) ClientException(org.opensearch.ad.common.exception.ClientException) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) ConnectException(java.net.ConnectException) EndRunException(org.opensearch.ad.common.exception.EndRunException) SinglePointFeatures(org.opensearch.ad.feature.SinglePointFeatures) InternalFailure(org.opensearch.ad.common.exception.InternalFailure) CommonName(org.opensearch.ad.constant.CommonName) Iterator(java.util.Iterator) FeatureManager(org.opensearch.ad.feature.FeatureManager) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) NotSerializedADExceptionName(org.opensearch.ad.common.exception.NotSerializedADExceptionName) INVALID_SEARCH_QUERY_MSG(org.opensearch.ad.constant.CommonErrorMessages.INVALID_SEARCH_QUERY_MSG) CompositeRetriever(org.opensearch.ad.feature.CompositeRetriever) PageIterator(org.opensearch.ad.feature.CompositeRetriever.PageIterator) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) CommonErrorMessages(org.opensearch.ad.constant.CommonErrorMessages) Entity(org.opensearch.ad.model.Entity) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ClusterService(org.opensearch.cluster.service.ClusterService) AnomalyDetectorPlugin(org.opensearch.ad.AnomalyDetectorPlugin) SearchPhaseExecutionException(org.opensearch.action.search.SearchPhaseExecutionException) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) StatNames(org.opensearch.ad.stats.StatNames) LogManager(org.apache.logging.log4j.LogManager) NetworkExceptionHelper(org.opensearch.common.transport.NetworkExceptionHelper) ParseUtils(org.opensearch.ad.util.ParseUtils) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) EndRunException(org.opensearch.ad.common.exception.EndRunException) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) NodeClosedException(org.opensearch.node.NodeClosedException) NodeNotConnectedException(org.opensearch.transport.NodeNotConnectedException) ActionNotFoundTransportException(org.opensearch.transport.ActionNotFoundTransportException) OpenSearchTimeoutException(org.opensearch.OpenSearchTimeoutException) ConnectTransportException(org.opensearch.transport.ConnectTransportException) ReceiveTimeoutTransportException(org.opensearch.transport.ReceiveTimeoutTransportException) ClientException(org.opensearch.ad.common.exception.ClientException) ConnectException(java.net.ConnectException) EndRunException(org.opensearch.ad.common.exception.EndRunException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) SearchPhaseExecutionException(org.opensearch.action.search.SearchPhaseExecutionException)

Example 2 with ClientException

use of org.opensearch.ad.common.exception.ClientException in project anomaly-detection by opensearch-project.

the class PreviewAnomalyDetectorTransportAction method previewExecute.

void previewExecute(PreviewAnomalyDetectorRequest request, ThreadContext.StoredContext context, ActionListener<PreviewAnomalyDetectorResponse> listener) {
    if (adCircuitBreakerService.isOpen()) {
        listener.onFailure(new LimitExceededException(request.getDetectorId(), CommonErrorMessages.MEMORY_CIRCUIT_BROKEN_ERR_MSG, false));
        return;
    }
    try {
        if (!lock.tryAcquire()) {
            listener.onFailure(new ClientException(request.getDetectorId(), CommonErrorMessages.REQUEST_THROTTLED_MSG));
            return;
        }
        try {
            AnomalyDetector detector = request.getDetector();
            String detectorId = request.getDetectorId();
            Instant startTime = request.getStartTime();
            Instant endTime = request.getEndTime();
            ActionListener<PreviewAnomalyDetectorResponse> releaseListener = ActionListener.runAfter(listener, () -> lock.release());
            if (detector != null) {
                String error = validateDetector(detector);
                if (StringUtils.isNotBlank(error)) {
                    listener.onFailure(new OpenSearchStatusException(error, RestStatus.BAD_REQUEST));
                    lock.release();
                    return;
                }
                anomalyDetectorRunner.executeDetector(detector, startTime, endTime, context, getPreviewDetectorActionListener(releaseListener, detector));
            } else {
                previewAnomalyDetector(releaseListener, detectorId, detector, startTime, endTime, context);
            }
        } catch (Exception e) {
            logger.error("Fail to preview", e);
            lock.release();
        }
    } catch (Exception e) {
        logger.error(e);
        listener.onFailure(e);
    }
}
Also used : Instant(java.time.Instant) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ClientException(org.opensearch.ad.common.exception.ClientException) AnomalyDetector(org.opensearch.ad.model.AnomalyDetector) OpenSearchStatusException(org.opensearch.OpenSearchStatusException) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) ClientException(org.opensearch.ad.common.exception.ClientException) IOException(java.io.IOException) OpenSearchStatusException(org.opensearch.OpenSearchStatusException)

Aggregations

AnomalyDetectionException (org.opensearch.ad.common.exception.AnomalyDetectionException)2 ClientException (org.opensearch.ad.common.exception.ClientException)2 LimitExceededException (org.opensearch.ad.common.exception.LimitExceededException)2 IOException (java.io.IOException)1 ConnectException (java.net.ConnectException)1 Instant (java.time.Instant)1 ArrayList (java.util.ArrayList)1 HashSet (java.util.HashSet)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Locale (java.util.Locale)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Optional (java.util.Optional)1 Set (java.util.Set)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 Collectors (java.util.stream.Collectors)1 LogManager (org.apache.logging.log4j.LogManager)1 Logger (org.apache.logging.log4j.Logger)1