use of org.opensearch.ad.transport.ADBatchAnomalyResultResponse in project anomaly-detection by opensearch-project.
the class ADBatchTaskRunner method startADBatchTaskOnWorkerNode.
/**
* Start AD task in dedicated batch task thread pool on worker node.
*
* @param adTask ad task
* @param runTaskRemotely run task remotely or not
* @param transportService transport service
* @param delegatedListener action listener
*/
public void startADBatchTaskOnWorkerNode(ADTask adTask, boolean runTaskRemotely, TransportService transportService, ActionListener<ADBatchAnomalyResultResponse> delegatedListener) {
try {
// check if cluster is eligible to run AD currently, if not eligible like
// circuit breaker open, will throw exception.
checkClusterState(adTask);
threadPool.executor(AD_BATCH_TASK_THREAD_POOL_NAME).execute(() -> {
ActionListener<String> internalListener = internalBatchTaskListener(adTask, transportService);
try {
executeADBatchTaskOnWorkerNode(adTask, internalListener);
} catch (Exception e) {
internalListener.onFailure(e);
}
});
delegatedListener.onResponse(new ADBatchAnomalyResultResponse(clusterService.localNode().getId(), runTaskRemotely));
} catch (Exception e) {
logger.error("Fail to start AD batch task " + adTask.getTaskId(), e);
delegatedListener.onFailure(e);
}
}
use of org.opensearch.ad.transport.ADBatchAnomalyResultResponse in project anomaly-detection by opensearch-project.
the class ADBatchTaskRunner method forwardOrExecuteADTask.
/**
* Forward AD task to work node.
* 1. For HC detector, return directly if no more pending entity. Otherwise check if
* there is AD task created for this entity. If yes, just forward the entity task
* to worker node; otherwise, create entity task first, then forward.
* 2. For single entity detector, set task as INIT state and forward task to worker
* node.
*
* @param adTask AD task
* @param transportService transport service
* @param listener action listener
*/
public void forwardOrExecuteADTask(ADTask adTask, TransportService transportService, ActionListener<ADBatchAnomalyResultResponse> listener) {
try {
checkIfADTaskCancelledAndCleanupCache(adTask);
String detectorId = adTask.getDetectorId();
AnomalyDetector detector = adTask.getDetector();
boolean isHCDetector = detector.isMultientityDetector();
if (isHCDetector) {
String entityString = adTaskCacheManager.pollEntity(detectorId);
logger.debug("Start to run entity: {} of detector {}", entityString, detectorId);
if (entityString == null) {
listener.onResponse(new ADBatchAnomalyResultResponse(clusterService.localNode().getId(), false));
return;
}
ActionListener<Object> wrappedListener = ActionListener.wrap(r -> logger.debug("Entity task created successfully"), e -> {
logger.error("Failed to start entity task for detector: {}, entity: {}", detectorId, entityString);
// If fail, move the entity into pending task queue
adTaskCacheManager.addPendingEntity(detectorId, entityString);
});
// This is to handle retry case. To retry entity, we need to get the old entity task created before.
Entity entity = adTaskManager.parseEntityFromString(entityString, adTask);
String parentTaskId = adTask.getTaskType().equals(ADTaskType.HISTORICAL_HC_ENTITY.name()) ? // For HISTORICAL_HC_ENTITY task, return its parent task id
adTask.getParentTaskId() : // For HISTORICAL_HC_DETECTOR task, its task id is parent task id
adTask.getTaskId();
adTaskManager.getAndExecuteOnLatestADTask(detectorId, parentTaskId, entity, ImmutableList.of(ADTaskType.HISTORICAL_HC_ENTITY), existingEntityTask -> {
if (existingEntityTask.isPresent()) {
// retry failed entity caused by limit exceed exception
// TODO: if task failed due to limit exceed exception in half way, resume from the break point or just clear
// the
// old AD tasks and rerun it? Currently we just support rerunning task failed due to limit exceed exception
// before starting.
ADTask adEntityTask = existingEntityTask.get();
logger.debug("Rerun entity task for task id: {}, error of last run: {}", adEntityTask.getTaskId(), adEntityTask.getError());
ActionListener<ADBatchAnomalyResultResponse> workerNodeResponseListener = workerNodeResponseListener(adEntityTask, transportService, listener);
forwardOrExecuteEntityTask(adEntityTask, transportService, workerNodeResponseListener);
} else {
logger.info("Create entity task for entity:{}", entityString);
Instant now = Instant.now();
ADTask adEntityTask = new ADTask.Builder().detectorId(adTask.getDetectorId()).detector(detector).isLatest(true).taskType(ADTaskType.HISTORICAL_HC_ENTITY.name()).executionStartTime(now).taskProgress(0.0f).initProgress(0.0f).state(ADTaskState.INIT.name()).initProgress(0.0f).lastUpdateTime(now).startedBy(adTask.getStartedBy()).coordinatingNode(clusterService.localNode().getId()).detectionDateRange(adTask.getDetectionDateRange()).user(adTask.getUser()).entity(entity).parentTaskId(parentTaskId).build();
adTaskManager.createADTaskDirectly(adEntityTask, r -> {
adEntityTask.setTaskId(r.getId());
ActionListener<ADBatchAnomalyResultResponse> workerNodeResponseListener = workerNodeResponseListener(adEntityTask, transportService, listener);
forwardOrExecuteEntityTask(adEntityTask, transportService, workerNodeResponseListener);
}, wrappedListener);
}
}, transportService, false, wrappedListener);
} else {
Map<String, Object> updatedFields = new HashMap<>();
updatedFields.put(STATE_FIELD, ADTaskState.INIT.name());
updatedFields.put(INIT_PROGRESS_FIELD, 0.0f);
ActionListener<ADBatchAnomalyResultResponse> workerNodeResponseListener = workerNodeResponseListener(adTask, transportService, listener);
adTaskManager.updateADTask(adTask.getTaskId(), updatedFields, ActionListener.wrap(r -> forwardOrExecuteEntityTask(adTask, transportService, workerNodeResponseListener), e -> {
workerNodeResponseListener.onFailure(e);
}));
}
} catch (Exception e) {
logger.error("Failed to forward or execute AD task " + adTask.getTaskId(), e);
listener.onFailure(e);
}
}
use of org.opensearch.ad.transport.ADBatchAnomalyResultResponse in project anomaly-detection by opensearch-project.
the class ADBatchTaskRunner method run.
/**
* Run AD task.
* 1. For HC detector, will get top entities first(initialize top entities). If top
* entities already initialized, will execute AD task directly.
* 2. For single entity detector, execute AD task directly.
* @param adTask single entity or HC detector task
* @param transportService transport service
* @param listener action listener
*/
public void run(ADTask adTask, TransportService transportService, ActionListener<ADBatchAnomalyResultResponse> listener) {
boolean isHCDetector = adTask.getDetector().isMultientityDetector();
if (isHCDetector && !adTaskCacheManager.topEntityInited(adTask.getDetectorId())) {
// Initialize top entities for HC detector
threadPool.executor(AD_BATCH_TASK_THREAD_POOL_NAME).execute(() -> {
ActionListener<ADBatchAnomalyResultResponse> hcDelegatedListener = getInternalHCDelegatedListener(adTask);
ActionListener<String> topEntitiesListener = getTopEntitiesListener(adTask, transportService, hcDelegatedListener);
try {
getTopEntities(adTask, topEntitiesListener);
} catch (Exception e) {
topEntitiesListener.onFailure(e);
}
});
listener.onResponse(new ADBatchAnomalyResultResponse(clusterService.localNode().getId(), false));
} else {
// Execute AD task for single entity detector or HC detector which top entities initialized
forwardOrExecuteADTask(adTask, transportService, listener);
}
}
Aggregations