use of cz.metacentrum.perun.taskslib.model.Task in project perun by CESNET.
the class TaskSchedulerImpl method scheduleTask.
// TODO ensure dependant tasks with scope DESTINATION go to the same engine
@Override
public Boolean scheduleTask(Task task) {
ExecService execService = task.getExecService();
Facility facility = task.getFacility();
Date time = new Date(System.currentTimeMillis());
DispatcherQueue dispatcherQueue = null;
if (task.getStatus().equals(TaskStatus.PROCESSING) && !task.isPropagationForced()) {
log.debug("Task {} already processing, will not schedule again.", task.toString());
return true;
}
log.debug("Scheduling TASK " + task.toString());
try {
dispatcherQueue = schedulingPool.getQueueForTask(task);
log.debug("Task {} is assigned to queue {}", task.getId(), (dispatcherQueue == null) ? "null" : dispatcherQueue.getClientID());
} catch (InternalErrorException e) {
log.warn("Task {} is not assigned to any queue", task.getId());
}
// check if the engine is still registered
if (dispatcherQueue != null && !dispatcherQueuePool.isThereDispatcherQueueForClient(dispatcherQueue.getClientID())) {
dispatcherQueue = null;
}
if (dispatcherQueue == null) {
// where should we send the task?
dispatcherQueue = dispatcherQueuePool.getAvailableQueue();
if (dispatcherQueue != null) {
try {
schedulingPool.setQueueForTask(task, dispatcherQueue);
} catch (InternalErrorException e) {
log.error("Could not set client queue for task {}: {}", task.getId(), e.getMessage());
return true;
}
log.debug("Assigned new queue " + dispatcherQueue.getQueueName() + " to task " + task.getId());
} else {
// bad luck...
log.error("Task " + task.toString() + " has no engine assigned and there are no engines registered...");
return true;
}
}
log.debug("Facility to be processed: " + facility.getId() + ", ExecService to be processed: " + execService.getId());
Boolean abortTask = false;
try {
refetchTaskInformation(task);
List<Service> assignedServices = perun.getServicesManagerBl().getAssignedServices(perunSession, task.getFacility());
if (!assignedServices.contains(execService.getService())) {
log.debug("Task {} has no longer service {} assigned, aborting.", task.getId(), execService.getId());
abortTask = true;
}
} catch (FacilityNotExistsException e1) {
log.debug("Facility {} for task {} no longer exists, aborting", facility.getId(), task.getId());
abortTask = true;
} catch (ServiceNotExistsException e1) {
log.debug("Service {} for task {} no longer exists, aborting", execService.getId(), task.getId());
abortTask = true;
} catch (InternalErrorException e1) {
log.error("Error checking facility or exec service for updates, task will not run now: {}", e1.getMessage());
return true;
} catch (PrivilegeException e1) {
log.error("Error checking facility or exec service for updates, task will not run now: {}", e1.getMessage());
return true;
}
// GEN tasks that we depend on...
if (abortTask && execService.getExecServiceType() == ExecServiceType.GENERATE) {
// GEN tasks may be aborted immediately
abortTask(task);
return false;
}
// do not perform further checks for task that is going to be aborted
if (!abortTask) {
log.debug("Is the execService ID:" + execService.getId() + " enabled globally?");
if (execService.isEnabled()) {
log.debug(" Yes, it is globally enabled.");
} else {
log.debug(" No, execService ID: " + execService.getId() + " is not enabled globally. Task will not run.");
return true;
}
log.debug(" Is the execService ID: " + execService.getId() + " denied on facility ID:" + facility.getId() + "?");
try {
if (!denialsResolver.isExecServiceDeniedOnFacility(execService, facility)) {
log.debug(" No, it is not.");
} else {
log.debug(" Yes, the execService ID: " + execService.getId() + " is denied on facility ID: " + facility.getId() + ". Task will not run.");
return true;
}
} catch (InternalErrorException e) {
log.error("Error getting disabled status for execService, task will not run now.");
return true;
}
}
List<ExecService> dependantServices = null;
List<Pair<ExecService, DependencyScope>> dependencies = null;
// If any of the ExecServices that depends on this one is running
// PROCESSING
// we will put the ExecService,Facility pair back to the pool.
// #################################################################################
log.debug(" Is there any execService that depends on [" + execService.getId() + "] in \"PROCESSING\" state?");
dependantServices = dependenciesResolver.listDependantServices(execService);
boolean proceed = true;
for (ExecService dependantService : dependantServices) {
Task dependantServiceTask = schedulingPool.getTask(dependantService, facility);
if (dependantServiceTask != null) {
if (dependantServiceTask.getStatus().equals(TaskStatus.PROCESSING)) {
log.debug(" There is a service [" + dependantServiceTask.getId() + "] running that depends on this one [" + execService + "], so we put this to sleep...");
// schedulingPool.addToPool(new Pair<ExecService,
// Facility>(execService, facility));
proceed = false;
break;
}
// This is probably wrong, so commenting out:
// 1) if there is some dispatcher queue active at the moment, dispatcherQueue is not null
// 2) if there is no dispatcher queue available, no point in setting the same queue another task has
// 3) the dependency should be handled the other way round, from dependant to dependent
/*
* try {
* if (dispatcherQueue == null &&
* schedulingPool.getQueueForTask(dependantServiceTask) != null) {
* schedulingPool.setQueueForTask(task, schedulingPool.getQueueForTask(dependantServiceTask));
* }
* } catch (InternalErrorException e) {
* log.debug(" Failed to set destination queue for task. This is weird, aborting.");
* proceed = false;
*}
*/
}
}
if (proceed) {
log.debug(" No, it is not. No dependent service is running, we can proceed.");
// If it is an ExecService of type SEND, we have to check its
// dependencies.
// We can skip this for GENERATE type (it has no dependencies by
// design).
// ########################################################################
log.debug(" Check whether the execService [" + execService.getId() + "] is of type SEND");
if (execService.getExecServiceType().equals(ExecServiceType.SEND)) {
log.debug(" Well, it is, so we have to check it's dependencies.");
// We check the status of all the ExecServices this ExecService
// depends on.
//
//
// Current approach disregards any SEND/GENERATE differences.
// Dependency on a GENERATE service is being treated as same as
// any other SEND dependency
// but for a small exception regarding ERROR and DONE states,
// see below:
//
// If the dependency is in one of the following states, we do:
// NONE Schedule it and wait (put this [ExecService,Facility]
// pair back to the SchedulingPool for a while).
// PROCESSING Wait
// ERROR IF dependency is GENERATE THEN DO
// Schedule it and wait (put this [ExecService,Facility] pair
// back to the SchedulingPool for a while).
// ELSE IF dependency is SEND THEN DO
// End with ERROR. (no point in trying, something is probably
// amiss on destination nodes...)
// ELSE
// throw new IllegalArgumentException
// FI
// DONE IF dependency is GENERATE THEN DO
// Schedule it and wait (put this [ExecService,Facility] pair
// back to the SchedulingPool for a while).
//
// It might look like we get an infinite loop where GENERATE
// will be in DONE and then rescheduled again and again.
// It is not so because PropagationMaintainer sets its state to
// NONE as soon as the SEND, that depends on it,
// enters either DONE or ERROR states (one of its finite
// states).
// ELSE IF dependency is SEND THEN DO
// Proceed (Yes, no need to schedule this dependency, it is done
// already and we don't care for how long it has been so at this
// point.)
// ELSE
// throw new IllegalArgumentException
// FI
// :-)
// #######################################################################################################
proceed = true;
dependencies = dependenciesResolver.listDependenciesAndScope(execService);
log.debug(" We are about to loop over execService [" + execService.getId() + "] dependencies.");
log.debug(" Number of dependencies:" + dependencies);
DispatcherQueue dependencyQueue = null;
for (Pair<ExecService, DependencyScope> dependencyPair : dependencies) {
ExecService dependency = dependencyPair.getLeft();
DependencyScope dependencyScope = dependencyPair.getRight();
Task dependencyServiceTask = schedulingPool.getTask(dependency, facility);
if (dependencyServiceTask == null) {
if (abortTask) {
log.info(" Task {} is going to be aborted, the dependency exec service {} will not be scheduled now.", task.getId(), dependency.getId());
} else {
// Dependency being NULL is equivalent to being in NONE
// state.
log.info(" Last Task [dependency:" + dependency.getId() + ", facility:" + facility.getId() + "] was NULL, we are gonna propagate.");
if (scheduleItAndWait(dependency, facility, execService, dispatcherQueue, time)) {
// task sucessfully scheduled, nothing to do
} else {
// TODO: task aborted - maybe set this one to error?
}
}
proceed = false;
} else {
boolean wasDependencyServiceTaskForced = dependencyServiceTask.isPropagationForced();
dependencyServiceTask.setPropagationForced(task.isPropagationForced());
switch(dependencyServiceTask.getStatus()) {
case DONE:
switch(dependency.getExecServiceType()) {
case GENERATE:
if (task.isSourceUpdated()) {
// we need to reschedule the GEN task as the source data has changed
log.debug(" Dependency ID " + dependency.getId() + " is in DONE and is going to be rescheduled as we need fresh data.");
rescheduleTask(dependencyServiceTask, execService, dispatcherQueue);
proceed = false;
} else {
log.debug(" Dependency ID " + dependency.getId() + " is in DONE and it is of type GENERATE, we can proceed.");
// Nothing, we can proceed...
try {
dependencyQueue = schedulingPool.getQueueForTask(dependencyServiceTask);
} catch (InternalErrorException e) {
log.error("Could not get queue for task {}", dependencyServiceTask.getId());
}
}
break;
case SEND:
log.debug(" Dependency ID " + dependencyServiceTask.getId() + " is in DONE and it is of type SEND, we can proceed.");
// Nothing, we can proceed...
break;
default:
throw new IllegalArgumentException("Unknown ExecService type. Expected GENERATE or SEND.");
}
break;
case ERROR:
switch(dependency.getExecServiceType()) {
case GENERATE:
log.info(" Dependency ID " + dependencyServiceTask.getId() + " is in ERROR and it is of type GENERATE, we are gonna propagate.");
// scheduleItAndWait(dependency, facility,
// execService, dispatcherQueue);
// try to run the generate task again
rescheduleTask(dependencyServiceTask, execService, dispatcherQueue);
proceed = false;
break;
case SEND:
log.info(" Dependency ID " + dependencyServiceTask.getId() + " is in ERROR and it is of type SEND, we are gonna end with ERROR.");
proceed = false;
// We end Task with error immediately.
schedulingPool.setTaskStatus(task, TaskStatus.ERROR);
// facility);
break;
default:
throw new IllegalArgumentException("Unknown ExecService type. Expected GENERATE or SEND.");
}
break;
case NONE:
log.info(" Last Task {} [dependency:" + dependency.getId() + ", facility:" + facility.getId() + "] was NONE, we are gonna propagate.", dependencyServiceTask.getId());
rescheduleTask(dependencyServiceTask, execService, dispatcherQueue);
proceed = false;
break;
case PLANNED:
log.info(" Dependency ID " + dependencyServiceTask.getId() + " is in PLANNED so we are gonna wait.");
// justWait(facility, execService);
if (dependencyScope.equals(DependencyScope.SERVICE)) {
proceed = false;
} else {
try {
dependencyQueue = schedulingPool.getQueueForTask(dependencyServiceTask);
} catch (InternalErrorException e) {
log.error("Could not get queue for task {}", dependencyServiceTask.getId());
}
}
break;
case PROCESSING:
log.info(" Dependency ID " + dependencyServiceTask.getId() + " is in PROCESSING so we are gonna wait.");
// justWait(facility, execService);
if (dependencyScope.equals(DependencyScope.SERVICE)) {
proceed = false;
} else {
try {
dependencyQueue = schedulingPool.getQueueForTask(dependencyServiceTask);
} catch (InternalErrorException e) {
log.error("Could not get queue for task {}", dependencyServiceTask.getId());
}
}
if (dependencyServiceTask.isPropagationForced() && !wasDependencyServiceTaskForced) {
// reschedule dependant only if originally was not forced !!!
rescheduleTask(dependencyServiceTask, execService, dispatcherQueue);
// XXX - should we proceed here?
}
break;
default:
throw new IllegalArgumentException("Unknown Task status. Expected DONE, ERROR, NONE, PLANNED or PROCESSING.");
}
}
}
// #########################################
if (proceed) {
if (abortTask) {
// the SEND task is going to be aborted now
abortTask(task);
return false;
} else {
if (dependencyQueue != null && dependencyQueue != dispatcherQueue) {
log.debug("Changing task {} destination queue to {} to match dependency task", task.getId(), dependencyQueue.getClientID());
try {
schedulingPool.setQueueForTask(task, dependencyQueue);
} catch (InternalErrorException e) {
log.error("Could not change task {} destination queue: {}", task.getId(), e.getMessage());
}
}
log.info(" SCHEDULING task [" + task.getId() + "], execService [" + execService.getId() + "] facility [" + facility.getId() + "] as PLANNED.");
task.setSchedule(time);
schedulingPool.setTaskStatus(task, TaskStatus.PLANNED);
sendToEngine(task);
}
// manipulateTasks(execService, facility, task);
} else {
if (abortTask) {
// the SEND task is going to be aborted now
abortTask(task);
return false;
} else {
// If we can not proceed, we just end here.
// ########################################
// The current ExecService,Facility pair should be sleeping
// in SchedulingPool at the moment...
log.info(" Task {} state set to NONE, will be scheduled again at the next cycle.", task.getId());
schedulingPool.setTaskStatus(task, TaskStatus.NONE);
}
}
} else if (execService.getExecServiceType().equals(ExecServiceType.GENERATE)) {
log.debug(" Well, it is not. ExecService of type GENERATE does not have any dependencies by design, so we schedule it immediately.");
log.info(" SCHEDULING task [" + task.getId() + "], execService [" + execService.getId() + "] facility [" + facility.getId() + "] as PLANNED.");
task.setSchedule(time);
schedulingPool.setTaskStatus(task, TaskStatus.PLANNED);
sendToEngine(task);
// manipulateTasks(execService, facility, task);
} else {
throw new IllegalArgumentException("Unknown ExecService type. Expected GENERATE or SEND.");
}
} else {
log.debug(" We do not proceed, we put the task [" + task.getId() + "], [" + execService.getId() + "] execService to sleep.");
}
return true;
}
use of cz.metacentrum.perun.taskslib.model.Task in project perun by CESNET.
the class TaskSchedulerImpl method processPool.
@Override
public void processPool() throws InternalErrorException {
initPerunSession();
log.debug("pool contains " + schedulingPool.getSize() + " tasks in total");
log.debug(" " + schedulingPool.getWaitingTasks().size() + " tasks are going to be processed");
for (Task task : schedulingPool.getWaitingTasks()) {
if (task.getExecService().getExecServiceType().equals(ExecServiceType.SEND)) {
scheduleTask(task);
}
// GEN tasks are scheduled only as dependencies
}
}
use of cz.metacentrum.perun.taskslib.model.Task in project perun by CESNET.
the class TaskExecutorEngineImpl method beginExecuting.
@Override
public void beginExecuting() {
int currentlyRunningGenTasks = 0;
// TODO count gen tasks when they run and finish
for (Task task : schedulingPool.getProcessingTasks()) {
if (task.getExecService().getExecServiceType().equals(ExecServiceType.GENERATE)) {
currentlyRunningGenTasks++;
}
}
log.debug("There are " + currentlyRunningGenTasks + " running gen tasks.");
if (currentlyRunningGenTasks >= MAX_RUNNING_GEN) {
log.warn("Reached the maximum number of concurrently running gen tasks.");
}
// run all tasks in scheduled state
Date now = new Date(System.currentTimeMillis());
for (Task task : schedulingPool.getPlannedTasks()) {
/*
if(schedulingPool.getProcessingTasks().size() > MAX_RUNNING) {
log.warn("Reached the maximum number of concurrently running tasks.");
break;
}
if(currentlyRunningGenTasks >= MAX_RUNNING_GEN) {
continue;
}
*/
log.debug("TASK " + task.toString() + " is to be run at " + task.getSchedule() + ", now is " + now);
if (task.getSchedule().before(now)) {
log.debug("TASK " + task.toString() + " is going to run");
runTask(task);
}
}
/*
* int executorWorkersCreated = 0;
* log.debug("Begin execution process..."); List<Task> tasks = null;
* Date olderThen = null; Date youngerThen = null; long hour = 3600000;
* long year = hour * 8760; olderThen = new
* Date(System.currentTimeMillis() - year); youngerThen = new
* Date(System.currentTimeMillis() + hour); tasks =
* taskManager.listTasksScheduledBetweenDates(olderThen, youngerThen,
* Integer.parseInt(propertiesBean.getProperty("engine.unique.id"))); if
* (tasks != null) { log.info("There are " + tasks.size() +
* " tasks between dates [" + olderThen + "] and [" + youngerThen +
* "]"); for (Task task : tasks) { // We are about to execute only these
* tasks that are in the PLANNED state. if
* (task.getStatus().equals(TaskStatus.PLANNED)) { ExecService
* execService = task.getExecService();
*
* // set task to PROCESSING state
* task.setStatus(TaskStatus.PROCESSING); task.setStartTime(new
* Date(System.currentTimeMillis())); taskManager.updateTask(task,
* Integer.parseInt(propertiesBean.getProperty("engine.unique.id"))); //
* TODO switch the state when following core failed somewhere // (for
* example when break is called)
*
* if
* (execService.getExecServiceType().equals(ExecServiceType.GENERATE)) {
* log.debug("I'm gonna create worker for ExecService GENERATE, ID:" +
* execService.getId()); ExecutorEngineWorker executorEngineWorker =
* createExecutorEngineWorker(); executorWorkersCreated++;
* executorEngineWorker.setTask(task);
* executorEngineWorker.setExecService(execService); //A dedicated
* executor for GENERATE
* taskExecutorGenWorkers.execute(executorEngineWorker); } else if
* (execService.getExecServiceType().equals(ExecServiceType.SEND)) {
* log.debug( "I'm gonna create worker for ExecService SEND, ID:" +
* execService.getId()); switch(task.getType()) { case SERIAL:
* executorWorkersCreated = startSerialWorkers(task); break; case
* PARALLEL: executorWorkersCreated = startParallelWorkers(task); break;
* default: break; } if(executorWorkersCreated < 0) { continue; } } else
* { throw new IllegalArgumentException(
* "ExecService type has to be either SEND or GENERATE."); } } } } else
* { log.debug("There are no tasks between dates [" + olderThen +
* "] and [" + youngerThen + "]"); }
* log.info("Executing process ended. Created workers:" +
* executorWorkersCreated);
*/
}
use of cz.metacentrum.perun.taskslib.model.Task in project perun by CESNET.
the class TaskSchedulerImpl method processPool.
@Override
public void processPool() throws InternalErrorException {
for (Task task : schedulingPool.getNewTasks()) {
if (task.isPropagationForced()) {
// this can happen only as a race between this thread and thread created for force-scheduled task
log.info("Found force-scheduled task {} during normal schedule cycle, resetting force flag", task.getId());
// may not be the best behavior...
task.setPropagationForced(false);
}
log.debug("Propagating task {}, ExecService:Facility : " + task.getExecServiceId() + ":" + task.getFacilityId(), task.getId());
propagateService(task, new Date(System.currentTimeMillis()));
}
}
use of cz.metacentrum.perun.taskslib.model.Task in project perun by CESNET.
the class EngineManagerImpl method switchUnfinishedTasksToERROR.
@Override
public void switchUnfinishedTasksToERROR() {
log.info("I am going to switched all unfinished tasks to ERROR and finished GEN tasks which data wasn't send to ERROR as well");
/* we set everything found to error to report it back to dispatcher */
for (Task task : schedulingPool.getDoneTasks()) {
ExecService execService = task.getExecService();
if (execService.getExecServiceType().equals(ExecServiceType.GENERATE)) {
log.debug("Setting task " + task.toString() + " to ERROR");
schedulingPool.setTaskStatus(task, TaskStatus.ERROR);
}
}
for (Task task : schedulingPool.getProcessingTasks()) {
log.debug("Setting task " + task.toString() + " to ERROR");
schedulingPool.setTaskStatus(task, TaskStatus.ERROR);
}
for (Task task : schedulingPool.getPlannedTasks()) {
log.debug("Setting task " + task.toString() + " to ERROR");
schedulingPool.setTaskStatus(task, TaskStatus.ERROR);
}
log.info("I'm done with it.");
}
Aggregations