use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.
the class EntityColdStarterTests method setUp.
@SuppressWarnings("unchecked")
@Override
public void setUp() throws Exception {
super.setUp();
numMinSamples = AnomalyDetectorSettings.NUM_MIN_SAMPLES;
clock = mock(Clock.class);
when(clock.instant()).thenReturn(Instant.now());
threadPool = mock(ThreadPool.class);
setUpADThreadPool(threadPool);
settings = Settings.EMPTY;
Client client = mock(Client.class);
clientUtil = mock(ClientUtil.class);
detector = TestHelpers.AnomalyDetectorBuilder.newInstance().setDetectionInterval(new IntervalTimeConfiguration(1, ChronoUnit.MINUTES)).setCategoryFields(ImmutableList.of(randomAlphaOfLength(5))).build();
job = TestHelpers.randomAnomalyDetectorJob(true, Instant.ofEpochMilli(1602401500000L), null);
doAnswer(invocation -> {
GetRequest request = invocation.getArgument(0);
ActionListener<GetResponse> listener = invocation.getArgument(2);
if (request.index().equals(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX)) {
listener.onResponse(TestHelpers.createGetResponse(job, detectorId, AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX));
} else {
listener.onResponse(TestHelpers.createGetResponse(detector, detectorId, AnomalyDetector.ANOMALY_DETECTORS_INDEX));
}
return null;
}).when(clientUtil).asyncRequest(any(GetRequest.class), any(), any(ActionListener.class));
Set<Setting<?>> nodestateSetting = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
nodestateSetting.add(MAX_RETRY_FOR_UNRESPONSIVE_NODE);
nodestateSetting.add(BACKOFF_MINUTES);
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, nodestateSetting);
DiscoveryNode discoveryNode = new DiscoveryNode("node1", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, clusterSettings);
stateManager = new NodeStateManager(client, xContentRegistry(), settings, clientUtil, clock, AnomalyDetectorSettings.HOURLY_MAINTENANCE, clusterService);
SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = new IntegerSensitiveSingleFeatureLinearUniformInterpolator();
interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator);
searchFeatureDao = mock(SearchFeatureDao.class);
checkpoint = mock(CheckpointDao.class);
featureManager = new FeatureManager(searchFeatureDao, interpolator, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.TRAIN_SAMPLE_TIME_RANGE_IN_HOURS, AnomalyDetectorSettings.MIN_TRAIN_SAMPLES, AnomalyDetectorSettings.MAX_SHINGLE_PROPORTION_MISSING, AnomalyDetectorSettings.MAX_IMPUTATION_NEIGHBOR_DISTANCE, AnomalyDetectorSettings.PREVIEW_SAMPLE_RATE, AnomalyDetectorSettings.MAX_PREVIEW_SAMPLES, AnomalyDetectorSettings.HOURLY_MAINTENANCE, threadPool, AnomalyDetectorPlugin.AD_THREAD_POOL_NAME);
checkpointWriteQueue = mock(CheckpointWriteWorker.class);
rcfSeed = 2051L;
entityColdStarter = new EntityColdStarter(clock, threadPool, stateManager, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.TIME_DECAY, numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, interpolator, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, settings, AnomalyDetectorSettings.HOURLY_MAINTENANCE, checkpointWriteQueue, rcfSeed, AnomalyDetectorSettings.MAX_COLD_START_ROUNDS);
detectorId = "123";
modelId = "123_entity_abc";
entityName = "abc";
priority = 0.3f;
entity = Entity.createSingleAttributeEntity("field", entityName);
released = new AtomicBoolean();
inProgressLatch = new CountDownLatch(1);
releaseSemaphore = () -> {
released.set(true);
inProgressLatch.countDown();
};
listener = ActionListener.wrap(releaseSemaphore);
modelManager = new ModelManager(mock(CheckpointDao.class), mock(Clock.class), AnomalyDetectorSettings.NUM_TREES, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE, AnomalyDetectorSettings.TIME_DECAY, AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, AnomalyDetectorSettings.MIN_PREVIEW_SIZE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, AnomalyDetectorSettings.HOURLY_MAINTENANCE, entityColdStarter, mock(FeatureManager.class), mock(MemoryTracker.class));
}
use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.
the class ADBatchTaskRunner method getDateRangeOfSourceData.
private void getDateRangeOfSourceData(ADTask adTask, BiConsumer<Long, Long> consumer, ActionListener<String> internalListener) {
String taskId = adTask.getTaskId();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().aggregation(AggregationBuilders.min(AGG_NAME_MIN_TIME).field(adTask.getDetector().getTimeField())).aggregation(AggregationBuilders.max(AGG_NAME_MAX_TIME).field(adTask.getDetector().getTimeField())).size(0);
if (adTask.getEntity() != null && adTask.getEntity().getAttributes().size() > 0) {
BoolQueryBuilder query = new BoolQueryBuilder();
adTask.getEntity().getAttributes().entrySet().forEach(entity -> query.filter(new TermQueryBuilder(entity.getKey(), entity.getValue())));
searchSourceBuilder.query(query);
}
SearchRequest request = new SearchRequest().indices(adTask.getDetector().getIndices().toArray(new String[0])).source(searchSourceBuilder);
client.search(request, ActionListener.wrap(r -> {
InternalMin minAgg = r.getAggregations().get(AGG_NAME_MIN_TIME);
InternalMax maxAgg = r.getAggregations().get(AGG_NAME_MAX_TIME);
double minValue = minAgg.getValue();
double maxValue = maxAgg.getValue();
// If time field not exist or there is no value, will return infinity value
if (minValue == Double.POSITIVE_INFINITY) {
internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the time field"));
return;
}
long interval = ((IntervalTimeConfiguration) adTask.getDetector().getDetectionInterval()).toDuration().toMillis();
DetectionDateRange detectionDateRange = adTask.getDetectionDateRange();
long dataStartTime = detectionDateRange.getStartTime().toEpochMilli();
long dataEndTime = detectionDateRange.getEndTime().toEpochMilli();
long minDate = (long) minValue;
long maxDate = (long) maxValue;
if (minDate >= dataEndTime || maxDate <= dataStartTime) {
internalListener.onFailure(new ResourceNotFoundException(adTask.getDetectorId(), "There is no data in the detection date range"));
return;
}
if (minDate > dataStartTime) {
dataStartTime = minDate;
}
if (maxDate < dataEndTime) {
dataEndTime = maxDate;
}
// normalize start/end time to make it consistent with feature data agg result
dataStartTime = dataStartTime - dataStartTime % interval;
dataEndTime = dataEndTime - dataEndTime % interval;
logger.debug("adjusted date range: start: {}, end: {}, taskId: {}", dataStartTime, dataEndTime, taskId);
if ((dataEndTime - dataStartTime) < NUM_MIN_SAMPLES * interval) {
internalListener.onFailure(new AnomalyDetectionException("There is not enough data to train model").countedInStats(false));
return;
}
consumer.accept(dataStartTime, dataEndTime);
}, e -> {
internalListener.onFailure(e);
}));
}
use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.
the class IndexAnomalyDetectorJobActionHandler method createJob.
private void createJob(AnomalyDetector detector) {
try {
IntervalTimeConfiguration interval = (IntervalTimeConfiguration) detector.getDetectionInterval();
Schedule schedule = new IntervalSchedule(Instant.now(), (int) interval.getInterval(), interval.getUnit());
Duration duration = Duration.of(interval.getInterval(), interval.getUnit());
AnomalyDetectorJob job = new AnomalyDetectorJob(detector.getDetectorId(), schedule, detector.getWindowDelay(), true, Instant.now(), null, Instant.now(), duration.getSeconds(), detector.getUser(), detector.getResultIndex());
getAnomalyDetectorJobForWrite(detector, job);
} catch (Exception e) {
String message = "Failed to parse anomaly detector job " + detectorId;
logger.error(message, e);
listener.onFailure(new OpenSearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR));
}
}
use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.
the class ModelValidationActionHandler method checkDataFilterSparsity.
private void checkDataFilterSparsity(long latestTime) {
AggregationBuilder aggregation = getBucketAggregation(latestTime, (IntervalTimeConfiguration) anomalyDetector.getDetectionInterval());
BoolQueryBuilder query = QueryBuilders.boolQuery().filter(anomalyDetector.getFilterQuery());
SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(query, aggregation);
SearchRequest searchRequest = new SearchRequest(anomalyDetector.getIndices().toArray(new String[0])).source(searchSourceBuilder);
client.search(searchRequest, ActionListener.wrap(response -> processDataFilterResults(response, latestTime), listener::onFailure));
}
use of org.opensearch.ad.model.IntervalTimeConfiguration in project anomaly-detection by opensearch-project.
the class ModelValidationActionHandler method checkFeatureQueryDelegate.
private void checkFeatureQueryDelegate(long latestTime) throws IOException {
ActionListener<MergeableList<double[]>> validateFeatureQueriesListener = ActionListener.wrap(response -> {
windowDelayRecommendation(latestTime);
}, exception -> {
listener.onFailure(new ADValidationException(exception.getMessage(), DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
});
MultiResponsesDelegateActionListener<MergeableList<double[]>> multiFeatureQueriesResponseListener = new MultiResponsesDelegateActionListener<>(validateFeatureQueriesListener, anomalyDetector.getFeatureAttributes().size(), CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, false);
for (Feature feature : anomalyDetector.getFeatureAttributes()) {
AggregationBuilder aggregation = getBucketAggregation(latestTime, (IntervalTimeConfiguration) anomalyDetector.getDetectionInterval());
BoolQueryBuilder query = QueryBuilders.boolQuery().filter(anomalyDetector.getFilterQuery());
List<String> featureFields = ParseUtils.getFieldNamesForFeature(feature, xContentRegistry);
for (String featureField : featureFields) {
query.filter(QueryBuilders.existsQuery(featureField));
}
SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(query, aggregation);
SearchRequest searchRequest = new SearchRequest(anomalyDetector.getIndices().toArray(new String[0])).source(searchSourceBuilder);
client.search(searchRequest, ActionListener.wrap(response -> {
Histogram aggregate = checkBucketResultErrors(response);
if (aggregate == null) {
return;
}
double fullBucketRate = processBucketAggregationResults(aggregate);
if (fullBucketRate < CONFIG_BUCKET_MINIMUM_SUCCESS_RATE) {
multiFeatureQueriesResponseListener.onFailure(new ADValidationException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, DetectorValidationIssueType.FEATURE_ATTRIBUTES, ValidationAspect.MODEL));
} else {
multiFeatureQueriesResponseListener.onResponse(new MergeableList<>(new ArrayList<>(Collections.singletonList(new double[] { fullBucketRate }))));
}
}, e -> {
logger.error(e);
multiFeatureQueriesResponseListener.onFailure(new OpenSearchStatusException(CommonErrorMessages.FEATURE_QUERY_TOO_SPARSE, RestStatus.BAD_REQUEST, e));
}));
}
}
Aggregations