Search in sources :

Example 1 with InjectSecurity

use of org.opensearch.commons.InjectSecurity in project anomaly-detection by opensearch-project.

the class AnomalyDetectorJobRunner method runAnomalyDetectionJob.

private void runAnomalyDetectionJob(AnomalyDetectorJob jobParameter, LockService lockService, LockModel lock, Instant detectionStartTime, Instant executionStartTime, String detectorId, String user, List<String> roles) {
    try (InjectSecurity injectSecurity = new InjectSecurity(detectorId, settings, client.threadPool().getThreadContext())) {
        // Injecting user role to verify if the user has permissions for our API.
        injectSecurity.inject(user, roles);
        AnomalyResultRequest request = new AnomalyResultRequest(detectorId, detectionStartTime.toEpochMilli(), executionStartTime.toEpochMilli());
        client.execute(AnomalyResultAction.INSTANCE, request, ActionListener.wrap(response -> {
            indexAnomalyResult(jobParameter, lockService, lock, detectionStartTime, executionStartTime, response);
        }, exception -> {
            handleAdException(jobParameter, lockService, lock, detectionStartTime, executionStartTime, exception);
        }));
    } catch (Exception e) {
        indexAnomalyResultException(jobParameter, lockService, lock, detectionStartTime, executionStartTime, e, true);
        log.error("Failed to execute AD job " + detectorId, e);
    }
}
Also used : ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADTaskState(org.opensearch.ad.model.ADTaskState) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) AnomalyDetectorFunction(org.opensearch.ad.rest.handler.AnomalyDetectorFunction) XContentParser(org.opensearch.common.xcontent.XContentParser) AD_THREAD_POOL_NAME(org.opensearch.ad.AnomalyDetectorPlugin.AD_THREAD_POOL_NAME) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) WriteRequest(org.opensearch.action.support.WriteRequest) ADIndex(org.opensearch.ad.indices.ADIndex) ActionListener(org.opensearch.action.ActionListener) GetResponse(org.opensearch.action.get.GetResponse) Client(org.opensearch.client.Client) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) Set(java.util.Set) XContentParserUtils.ensureExpectedToken(org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken) ADTaskManager(org.opensearch.ad.task.ADTaskManager) Settings(org.opensearch.common.settings.Settings) Instant(java.time.Instant) UPDATED(org.opensearch.action.DocWriteResponse.Result.UPDATED) List(java.util.List) AnomalyResultAction(org.opensearch.ad.transport.AnomalyResultAction) Logger(org.apache.logging.log4j.Logger) AnomalyResult(org.opensearch.ad.model.AnomalyResult) CAN_NOT_FIND_LATEST_TASK(org.opensearch.ad.constant.CommonErrorMessages.CAN_NOT_FIND_LATEST_TASK) CREATED(org.opensearch.action.DocWriteResponse.Result.CREATED) IntervalSchedule(org.opensearch.jobscheduler.spi.schedule.IntervalSchedule) XContentType(org.opensearch.common.xcontent.XContentType) InjectSecurity(org.opensearch.commons.InjectSecurity) JobExecutionContext(org.opensearch.jobscheduler.spi.JobExecutionContext) DiscoveryNodeFilterer(org.opensearch.ad.util.DiscoveryNodeFilterer) AnomalyIndexHandler(org.opensearch.ad.transport.handler.AnomalyIndexHandler) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) FeatureData(org.opensearch.ad.model.FeatureData) AnomalyDetectorSettings(org.opensearch.ad.settings.AnomalyDetectorSettings) ArrayList(java.util.ArrayList) ProfileRequest(org.opensearch.ad.transport.ProfileRequest) XCONTENT_WITH_TYPE(org.opensearch.ad.util.RestHandlerUtils.XCONTENT_WITH_TYPE) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) LockModel(org.opensearch.jobscheduler.spi.LockModel) ExecutorService(java.util.concurrent.ExecutorService) EndRunException(org.opensearch.ad.common.exception.EndRunException) LockService(org.opensearch.jobscheduler.spi.utils.LockService) InternalFailure(org.opensearch.ad.common.exception.InternalFailure) GetRequest(org.opensearch.action.get.GetRequest) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) AnomalyDetectionIndices(org.opensearch.ad.indices.AnomalyDetectionIndices) DetectorProfileName(org.opensearch.ad.model.DetectorProfileName) ScheduledJobParameter(org.opensearch.jobscheduler.spi.ScheduledJobParameter) AnomalyResultTransportAction(org.opensearch.ad.transport.AnomalyResultTransportAction) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) User(org.opensearch.commons.authuser.User) AnomalyResultResponse(org.opensearch.ad.transport.AnomalyResultResponse) AnomalyDetectorJob(org.opensearch.ad.model.AnomalyDetectorJob) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) ScheduledJobRunner(org.opensearch.jobscheduler.spi.ScheduledJobRunner) IndexRequest(org.opensearch.action.index.IndexRequest) IntervalTimeConfiguration(org.opensearch.ad.model.IntervalTimeConfiguration) LogManager(org.apache.logging.log4j.LogManager) ProfileAction(org.opensearch.ad.transport.ProfileAction) AnomalyResultRequest(org.opensearch.ad.transport.AnomalyResultRequest) InjectSecurity(org.opensearch.commons.InjectSecurity) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) EndRunException(org.opensearch.ad.common.exception.EndRunException) IOException(java.io.IOException)

Example 2 with InjectSecurity

use of org.opensearch.commons.InjectSecurity in project anomaly-detection by opensearch-project.

the class ADBatchTaskRunner method detectAnomaly.

private void detectAnomaly(ADTask adTask, Map<Long, Optional<double[]>> dataPoints, long pieceStartTime, long pieceEndTime, long dataStartTime, long dataEndTime, long interval, Instant executeStartTime, ActionListener<String> internalListener) {
    String taskId = adTask.getTaskId();
    ThresholdedRandomCutForest trcf = adTaskCacheManager.getTRcfModel(taskId);
    Deque<Map.Entry<Long, Optional<double[]>>> shingle = adTaskCacheManager.getShingle(taskId);
    List<AnomalyResult> anomalyResults = new ArrayList<>();
    long intervalEndTime = pieceStartTime;
    for (int i = 0; i < pieceSize && intervalEndTime < dataEndTime; i++) {
        Optional<double[]> dataPoint = dataPoints.containsKey(intervalEndTime) ? dataPoints.get(intervalEndTime) : Optional.empty();
        intervalEndTime = intervalEndTime + interval;
        SinglePointFeatures feature = featureManager.getShingledFeatureForHistoricalAnalysis(adTask.getDetector(), shingle, dataPoint, intervalEndTime);
        List<FeatureData> featureData = null;
        if (feature.getUnprocessedFeatures().isPresent()) {
            featureData = ParseUtils.getFeatureData(feature.getUnprocessedFeatures().get(), adTask.getDetector());
        }
        if (!feature.getProcessedFeatures().isPresent()) {
            String error = feature.getUnprocessedFeatures().isPresent() ? "No full shingle in current detection window" : "No data in current detection window";
            AnomalyResult anomalyResult = new AnomalyResult(adTask.getDetectorId(), adTask.getDetectorLevelTaskId(), featureData, Instant.ofEpochMilli(intervalEndTime - interval), Instant.ofEpochMilli(intervalEndTime), executeStartTime, Instant.now(), error, adTask.getEntity(), adTask.getDetector().getUser(), anomalyDetectionIndices.getSchemaVersion(ADIndex.RESULT), adTask.getEntityModelId());
            anomalyResults.add(anomalyResult);
        } else {
            double[] point = feature.getProcessedFeatures().get();
            // 0 is placeholder for timestamp. In the future, we will add
            // data time stamp there.
            AnomalyDescriptor descriptor = trcf.process(point, 0);
            double score = descriptor.getRCFScore();
            if (!adTaskCacheManager.isThresholdModelTrained(taskId) && score > 0) {
                adTaskCacheManager.setThresholdModelTrained(taskId, true);
            }
            AnomalyResult anomalyResult = AnomalyResult.fromRawTRCFResult(adTask.getDetectorId(), adTask.getDetector().getDetectorIntervalInMilliseconds(), adTask.getDetectorLevelTaskId(), score, descriptor.getAnomalyGrade(), descriptor.getDataConfidence(), featureData, Instant.ofEpochMilli(intervalEndTime - interval), Instant.ofEpochMilli(intervalEndTime), executeStartTime, Instant.now(), null, adTask.getEntity(), adTask.getDetector().getUser(), anomalyDetectionIndices.getSchemaVersion(ADIndex.RESULT), adTask.getEntityModelId(), modelManager.normalizeAttribution(trcf.getForest(), descriptor.getRelevantAttribution()), descriptor.getRelativeIndex(), descriptor.getPastValues(), descriptor.getExpectedValuesList(), descriptor.getLikelihoodOfValues(), descriptor.getThreshold());
            anomalyResults.add(anomalyResult);
        }
    }
    String user;
    List<String> roles;
    if (adTask.getUser() == null) {
        // It's possible that user create domain with security disabled, then enable security
        // after upgrading. This is for BWC, for old detectors which created when security
        // disabled, the user will be null.
        user = "";
        roles = settings.getAsList("", ImmutableList.of("all_access", "AmazonES_all_access"));
    } else {
        user = adTask.getUser().getName();
        roles = adTask.getUser().getRoles();
    }
    String resultIndex = adTask.getDetector().getResultIndex();
    if (resultIndex == null) {
        // if result index is null, store anomaly result directly
        storeAnomalyResultAndRunNextPiece(adTask, pieceEndTime, dataStartTime, dataEndTime, interval, internalListener, anomalyResults, resultIndex, null);
        return;
    }
    try (InjectSecurity injectSecurity = new InjectSecurity(adTask.getTaskId(), settings, client.threadPool().getThreadContext())) {
        // Injecting user role to verify if the user has permissions to write result to result index.
        injectSecurity.inject(user, roles);
        storeAnomalyResultAndRunNextPiece(adTask, pieceEndTime, dataStartTime, dataEndTime, interval, internalListener, anomalyResults, resultIndex, () -> injectSecurity.close());
    } catch (Exception exception) {
        logger.error("Failed to inject user roles", exception);
        internalListener.onFailure(exception);
    }
}
Also used : ArrayList(java.util.ArrayList) FeatureData(org.opensearch.ad.model.FeatureData) LimitExceededException(org.opensearch.ad.common.exception.LimitExceededException) ResourceNotFoundException(org.opensearch.ad.common.exception.ResourceNotFoundException) ADTaskCancelledException(org.opensearch.ad.common.exception.ADTaskCancelledException) AnomalyDetectionException(org.opensearch.ad.common.exception.AnomalyDetectionException) EndRunException(org.opensearch.ad.common.exception.EndRunException) SinglePointFeatures(org.opensearch.ad.feature.SinglePointFeatures) AnomalyDescriptor(com.amazon.randomcutforest.parkservices.AnomalyDescriptor) AnomalyResult(org.opensearch.ad.model.AnomalyResult) InjectSecurity(org.opensearch.commons.InjectSecurity) ThresholdedRandomCutForest(com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest)

Example 3 with InjectSecurity

use of org.opensearch.commons.InjectSecurity in project anomaly-detection by opensearch-project.

the class AnomalyDetectionIndices method validateCustomIndexForBackendJob.

public <T> void validateCustomIndexForBackendJob(String resultIndex, String securityLogId, String user, List<String> roles, AnomalyDetectorFunction function, ActionListener<T> listener) {
    if (!doesIndexExist(resultIndex)) {
        listener.onFailure(new EndRunException(CAN_NOT_FIND_RESULT_INDEX + resultIndex, true));
        return;
    }
    if (!isValidResultIndexMapping(resultIndex)) {
        listener.onFailure(new EndRunException("Result index mapping is not correct", true));
        return;
    }
    try (InjectSecurity injectSecurity = new InjectSecurity(securityLogId, settings, client.threadPool().getThreadContext())) {
        injectSecurity.inject(user, roles);
        ActionListener<T> wrappedListener = ActionListener.wrap(r -> {
            listener.onResponse(r);
        }, e -> {
            injectSecurity.close();
            listener.onFailure(e);
        });
        validateCustomResultIndexAndExecute(resultIndex, () -> {
            injectSecurity.close();
            function.execute();
        }, wrappedListener);
    } catch (Exception e) {
        logger.error("Failed to validate custom index for backend job " + securityLogId, e);
        listener.onFailure(e);
    }
}
Also used : EndRunException(org.opensearch.ad.common.exception.EndRunException) InjectSecurity(org.opensearch.commons.InjectSecurity) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) EndRunException(org.opensearch.ad.common.exception.EndRunException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) IOException(java.io.IOException)

Aggregations

EndRunException (org.opensearch.ad.common.exception.EndRunException)3 InjectSecurity (org.opensearch.commons.InjectSecurity)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 AnomalyDetectionException (org.opensearch.ad.common.exception.AnomalyDetectionException)2 ResourceNotFoundException (org.opensearch.ad.common.exception.ResourceNotFoundException)2 AnomalyResult (org.opensearch.ad.model.AnomalyResult)2 FeatureData (org.opensearch.ad.model.FeatureData)2 AnomalyDescriptor (com.amazon.randomcutforest.parkservices.AnomalyDescriptor)1 ThresholdedRandomCutForest (com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest)1 Throwables (com.google.common.base.Throwables)1 ImmutableList (com.google.common.collect.ImmutableList)1 Instant (java.time.Instant)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Set (java.util.Set)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 ExecutorService (java.util.concurrent.ExecutorService)1 LogManager (org.apache.logging.log4j.LogManager)1 Logger (org.apache.logging.log4j.Logger)1