Examples with DetectionDateRange org.opensearch.ad.model.DetectionDateRange used on opensource projects

Example 11 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class ADRestTestUtils method startHistoricalAnalysis.

/**
 * Start historical analysis.
 * For AD versions on or before 1.0, should pass historical detector id to
 * this function.
 * For AD version on or after 1.1, can pass any detector id to this function.
 *
 * @param client REST client
 * @param detectorId detector id
 * @return task id of historical analysis
 * @throws IOException exception may throw in toHttpEntity and entityAsMap
 */
@SuppressWarnings("unchecked")
public static String startHistoricalAnalysis(RestClient client, String detectorId) throws IOException {
    Instant now = Instant.now();
    DetectionDateRange dateRange = new DetectionDateRange(now.minus(30, ChronoUnit.DAYS), now);
    Response response = TestHelpers.makeRequest(client, "POST", TestHelpers.LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI + "/" + detectorId + "/_start", ImmutableMap.of(), // Need to pass detection date range in http body if need to start historical analysis.
    TestHelpers.toHttpEntity(TestHelpers.toJsonString(dateRange)), null);
    Map<String, Object> startDetectorResponseMap = entityAsMap(response);
    String taskId = (String) startDetectorResponseMap.get("_id");
    return taskId;
}

Example 12 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class AnomalyDetectorRestApiIT method testRunDetectorWithNoEnabledFeature.

public void testRunDetectorWithNoEnabledFeature() throws Exception {
    AnomalyDetector detector = createRandomAnomalyDetector(true, true, client(), false);
    Assert.assertNotNull(detector.getDetectorId());
    Instant now = Instant.now();
    ResponseException e = expectThrows(ResponseException.class, () -> startAnomalyDetector(detector.getDetectorId(), new DetectionDateRange(now.minus(10, ChronoUnit.DAYS), now), client()));
    assertTrue(e.getMessage().contains("Can't start detector job as no enabled features configured"));
}

Example 13 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class ADTaskManager method checkTaskSlots.

/**
 * Check available task slots before start historical analysis and scale task lane.
 * This check will be done on lead node which will gather detector task slots of all
 * data nodes and calculate how many task slots available.
 *
 * @param adTask AD task
 * @param detector detector
 * @param detectionDateRange detection date range
 * @param user user
 * @param afterCheckAction target task action to run after task slot checking
 * @param transportService transport service
 * @param listener action listener
 */
public void checkTaskSlots(ADTask adTask, AnomalyDetector detector, DetectionDateRange detectionDateRange, User user, ADTaskAction afterCheckAction, TransportService transportService, ActionListener<AnomalyDetectorJobResponse> listener) {
    String detectorId = detector.getDetectorId();
    logger.debug("Start checking task slots for detector: {}, task action: {}", detectorId, afterCheckAction);
    if (!checkingTaskSlot.tryAcquire()) {
        logger.info("Can't acquire checking task slot semaphore for detector {}", detectorId);
        listener.onFailure(new OpenSearchStatusException("Too many historical analysis requests in short time. Please retry later.", RestStatus.FORBIDDEN));
        return;
    }
    ActionListener<AnomalyDetectorJobResponse> wrappedActionListener = ActionListener.runAfter(listener, () -> {
        checkingTaskSlot.release(1);
        logger.debug("Release checking task slot semaphore on lead node for detector {}", detectorId);
    });
    hashRing.getNodesWithSameLocalAdVersion(nodes -> {
        int maxAdTaskSlots = nodes.length * maxAdBatchTaskPerNode;
        ADStatsRequest adStatsRequest = new ADStatsRequest(nodes);
        adStatsRequest.addAll(ImmutableSet.of(AD_USED_BATCH_TASK_SLOT_COUNT.getName(), AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT.getName()));
        client.execute(ADStatsNodesAction.INSTANCE, adStatsRequest, ActionListener.wrap(adStatsResponse -> {
            // Total entity tasks running on worker nodes
            int totalUsedTaskSlots = 0;
            // Total assigned task slots on coordinating nodes
            int totalAssignedTaskSlots = 0;
            for (ADStatsNodeResponse response : adStatsResponse.getNodes()) {
                totalUsedTaskSlots += (int) response.getStatsMap().get(AD_USED_BATCH_TASK_SLOT_COUNT.getName());
                totalAssignedTaskSlots += (int) response.getStatsMap().get(AD_DETECTOR_ASSIGNED_BATCH_TASK_SLOT_COUNT.getName());
            }
            logger.info("Current total used task slots is {}, total detector assigned task slots is {} when start historical " + "analysis for detector {}", totalUsedTaskSlots, totalAssignedTaskSlots, detectorId);
            // In happy case, totalAssignedTaskSlots >= totalUsedTaskSlots. If some coordinating node left, then we can't
            // get detector task slots cached on it, so it's possible that totalAssignedTaskSlots < totalUsedTaskSlots.
            int currentUsedTaskSlots = Math.max(totalUsedTaskSlots, totalAssignedTaskSlots);
            if (currentUsedTaskSlots >= maxAdTaskSlots) {
                wrappedActionListener.onFailure(new OpenSearchStatusException("No available task slot", RestStatus.BAD_REQUEST));
                return;
            }
            int availableAdTaskSlots = maxAdTaskSlots - currentUsedTaskSlots;
            logger.info("Current available task slots is {} for historical analysis of detector {}", availableAdTaskSlots, detectorId);
            if (ADTaskAction.SCALE_ENTITY_TASK_SLOTS == afterCheckAction) {
                forwardToCoordinatingNode(adTask, detector, detectionDateRange, user, afterCheckAction, transportService, wrappedActionListener, availableAdTaskSlots);
                return;
            }
            // It takes long time to check top entities especially for multi-category HC. Tested with
            // 1.8 billion docs for multi-category HC, it took more than 20 seconds and caused timeout.
            // By removing top entity check, it took about 200ms to return. So just remove it to make
            // sure REST API can return quickly.
            // We may assign more task slots. For example, cluster has 4 data nodes, each node can run 2
            // batch tasks, so the available task slot number is 8. If max running entities per HC is 4,
            // then we will assign 4 tasks slots to this HC detector (4 is less than 8). The data index
            // only has 2 entities. So we assign 2 more task slots than actual need. But it's ok as we
            // will auto tune task slot when historical analysis task starts.
            int approvedTaskSlots = detector.isMultientityDetector() ? Math.min(maxRunningEntitiesPerDetector, availableAdTaskSlots) : 1;
            forwardToCoordinatingNode(adTask, detector, detectionDateRange, user, afterCheckAction, transportService, wrappedActionListener, approvedTaskSlots);
        }, exception -> {
            logger.error("Failed to get node's task stats for detector " + detectorId, exception);
            wrappedActionListener.onFailure(exception);
        }));
    }, wrappedActionListener);
}

Example 14 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class ADBatchTaskRunner method getDateRangeOfSourceData.

private void getDateRangeOfSourceData(ADTask adTask, BiConsumer<Long, Long> consumer, ActionListener<String> internalListener) {
    String taskId = adTask.getTaskId();
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().aggregation(AggregationBuilders.min(AGG_NAME_MIN_TIME).field(adTask.getDetector().getTimeField())).aggregation(AggregationBuilders.max(AGG_NAME_MAX_TIME).field(adTask.getDetector().getTimeField())).size(0);
    if (adTask.getEntity() != null && adTask.getEntity().getAttributes().size() > 0) {
        BoolQueryBuilder query = new BoolQueryBuilder();
        adTask.getEntity().getAttributes().entrySet().forEach(entity -> query.filter(new TermQueryBuilder(entity.getKey(), entity.getValue())));
        searchSourceBuilder.query(query);
    }
    SearchRequest request = new SearchRequest().indices(adTask.getDetector().getIndices().toArray(new String[0])).source(searchSourceBuilder);
    client.search(request, ActionListener.wrap(r -> {
        InternalMin minAgg = r.getAggregations().get(AGG_NAME_MIN_TIME);
        InternalMax maxAgg = r.getAggregations().get(AGG_NAME_MAX_TIME);
        double minValue = minAgg.getValue();
        double maxValue = maxAgg.getValue();
        // If time field not exist or there is no value, will return infinity value
        if (minValue == Double.POSITIVE_INFINITY) {
            internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the time field"));
            return;
        }
        long interval = ((IntervalTimeConfiguration) adTask.getDetector().getDetectionInterval()).toDuration().toMillis();
        DetectionDateRange detectionDateRange = adTask.getDetectionDateRange();
        long dataStartTime = detectionDateRange.getStartTime().toEpochMilli();
        long dataEndTime = detectionDateRange.getEndTime().toEpochMilli();
        long minDate = (long) minValue;
        long maxDate = (long) maxValue;
        if (minDate >= dataEndTime || maxDate <= dataStartTime) {
            internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the detection date range"));
            return;
        }
        if (minDate > dataStartTime) {
            dataStartTime = minDate;
        }
        if (maxDate < dataEndTime) {
            dataEndTime = maxDate;
        }
        // normalize start/end time to make it consistent with feature data agg result
        dataStartTime = dataStartTime - dataStartTime % interval;
        dataEndTime = dataEndTime - dataEndTime % interval;
        logger.debug("adjusted date range: start: {}, end: {}, taskId: {}", dataStartTime, dataEndTime, taskId);
        if ((dataEndTime - dataStartTime) < NUM_MIN_SAMPLES * interval) {
            internalListener.onFailure(new AnomalyDetectionException("There is not enough data to train model").countedInStats(false));
            return;
        }
        consumer.accept(dataStartTime, dataEndTime);
    }, e -> {
        internalListener.onFailure(e);
    }));
}

Example 15 with DetectionDateRange

use of org.opensearch.ad.model.DetectionDateRange in project anomaly-detection by opensearch-project.

the class RestAnomalyDetectorJobAction method prepareRequest.

@Override
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
    if (!EnabledSetting.isADPluginEnabled()) {
        throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG);
    }
    String detectorId = request.param(DETECTOR_ID);
    long seqNo = request.paramAsLong(IF_SEQ_NO, SequenceNumbers.UNASSIGNED_SEQ_NO);
    long primaryTerm = request.paramAsLong(IF_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
    boolean historical = request.paramAsBoolean("historical", false);
    String rawPath = request.rawPath();
    DetectionDateRange detectionDateRange = parseDetectionDateRange(request);
    AnomalyDetectorJobRequest anomalyDetectorJobRequest = new AnomalyDetectorJobRequest(detectorId, detectionDateRange, historical, seqNo, primaryTerm, rawPath);
    return channel -> client.execute(AnomalyDetectorJobAction.INSTANCE, anomalyDetectorJobRequest, new RestToXContentListener<>(channel));
}