use of org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES in project anomaly-detection by opensearch-project.
the class ADTaskManager method updateLatestRealtimeTaskOnCoordinatingNode.
/**
* Update realtime task cache on realtime detector's coordinating node.
*
* @param detectorId detector id
* @param state new state
* @param rcfTotalUpdates rcf total updates
* @param detectorIntervalInMinutes detector interval in minutes
* @param error error
* @param listener action listener
*/
public void updateLatestRealtimeTaskOnCoordinatingNode(String detectorId, String state, Long rcfTotalUpdates, Long detectorIntervalInMinutes, String error, ActionListener<UpdateResponse> listener) {
Float initProgress = null;
String newState = null;
// calculate init progress and task state with RCF total updates
if (detectorIntervalInMinutes != null && rcfTotalUpdates != null) {
newState = ADTaskState.INIT.name();
if (rcfTotalUpdates < NUM_MIN_SAMPLES) {
initProgress = (float) rcfTotalUpdates / NUM_MIN_SAMPLES;
} else {
newState = ADTaskState.RUNNING.name();
initProgress = 1.0f;
}
}
// Check if new state is not null and override state calculated with rcf total updates
if (state != null) {
newState = state;
}
error = Optional.ofNullable(error).orElse("");
if (!adTaskCacheManager.isRealtimeTaskChanged(detectorId, newState, initProgress, error)) {
// If task not changed, no need to update, just return
listener.onResponse(null);
return;
}
Map<String, Object> updatedFields = new HashMap<>();
updatedFields.put(COORDINATING_NODE_FIELD, clusterService.localNode().getId());
if (initProgress != null) {
updatedFields.put(INIT_PROGRESS_FIELD, initProgress);
updatedFields.put(ESTIMATED_MINUTES_LEFT_FIELD, Math.max(0, NUM_MIN_SAMPLES - rcfTotalUpdates) * detectorIntervalInMinutes);
}
if (newState != null) {
updatedFields.put(STATE_FIELD, newState);
}
if (error != null) {
updatedFields.put(ERROR_FIELD, error);
}
Float finalInitProgress = initProgress;
// Variable used in lambda expression should be final or effectively final
String finalError = error;
String finalNewState = newState;
updateLatestADTask(detectorId, ADTaskType.REALTIME_TASK_TYPES, updatedFields, ActionListener.wrap(r -> {
logger.debug("Updated latest realtime AD task successfully for detector {}", detectorId);
adTaskCacheManager.updateRealtimeTaskCache(detectorId, finalNewState, finalInitProgress, finalError);
listener.onResponse(r);
}, e -> {
logger.error("Failed to update realtime task for detector " + detectorId, e);
listener.onFailure(e);
}));
}
use of org.opensearch.ad.settings.AnomalyDetectorSettings.NUM_MIN_SAMPLES in project anomaly-detection by opensearch-project.
the class ADBatchTaskRunner method getDateRangeOfSourceData.
private void getDateRangeOfSourceData(ADTask adTask, BiConsumer<Long, Long> consumer, ActionListener<String> internalListener) {
String taskId = adTask.getTaskId();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().aggregation(AggregationBuilders.min(AGG_NAME_MIN_TIME).field(adTask.getDetector().getTimeField())).aggregation(AggregationBuilders.max(AGG_NAME_MAX_TIME).field(adTask.getDetector().getTimeField())).size(0);
if (adTask.getEntity() != null && adTask.getEntity().getAttributes().size() > 0) {
BoolQueryBuilder query = new BoolQueryBuilder();
adTask.getEntity().getAttributes().entrySet().forEach(entity -> query.filter(new TermQueryBuilder(entity.getKey(), entity.getValue())));
searchSourceBuilder.query(query);
}
SearchRequest request = new SearchRequest().indices(adTask.getDetector().getIndices().toArray(new String[0])).source(searchSourceBuilder);
client.search(request, ActionListener.wrap(r -> {
InternalMin minAgg = r.getAggregations().get(AGG_NAME_MIN_TIME);
InternalMax maxAgg = r.getAggregations().get(AGG_NAME_MAX_TIME);
double minValue = minAgg.getValue();
double maxValue = maxAgg.getValue();
// If time field not exist or there is no value, will return infinity value
if (minValue == Double.POSITIVE_INFINITY) {
internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the time field"));
return;
}
long interval = ((IntervalTimeConfiguration) adTask.getDetector().getDetectionInterval()).toDuration().toMillis();
DetectionDateRange detectionDateRange = adTask.getDetectionDateRange();
long dataStartTime = detectionDateRange.getStartTime().toEpochMilli();
long dataEndTime = detectionDateRange.getEndTime().toEpochMilli();
long minDate = (long) minValue;
long maxDate = (long) maxValue;
if (minDate >= dataEndTime || maxDate <= dataStartTime) {
internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the detection date range"));
return;
}
if (minDate > dataStartTime) {
dataStartTime = minDate;
}
if (maxDate < dataEndTime) {
dataEndTime = maxDate;
}
// normalize start/end time to make it consistent with feature data agg result
dataStartTime = dataStartTime - dataStartTime % interval;
dataEndTime = dataEndTime - dataEndTime % interval;
logger.debug("adjusted date range: start: {}, end: {}, taskId: {}", dataStartTime, dataEndTime, taskId);
if ((dataEndTime - dataStartTime) < NUM_MIN_SAMPLES * interval) {
internalListener.onFailure(new AnomalyDetectionException("There is not enough data to train model").countedInStats(false));
return;
}
consumer.accept(dataStartTime, dataEndTime);
}, e -> {
internalListener.onFailure(e);
}));
}
Aggregations