Search in sources :

Example 31 with WorkItem

use of org.batfish.common.WorkItem in project batfish by batfish.

the class WorkQueueMgr method queueParsingWork.

private synchronized boolean queueParsingWork(QueuedWork work) throws Exception {
    WorkItem wItem = work.getWorkItem();
    WorkDetails wDetails = work.getDetails();
    // if incomplete work for this testrig exists, lets just reject this parsing work.
    // parsing work cannot proceeed in parallel because it may overwrite files used by others.
    // instead of rejecting, we could have queued it as BLOCKED but we risk cycles of BLOCKED work
    // this should not be a common case anyway, so we aren't losing much by rejecting it
    QueuedWork incompleteWork = getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, null);
    if (incompleteWork != null) {
        throw new BatfishException("Cannot queue parsing work while other work is incomplete");
    } else {
        EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv);
        if (envMetadata.getProcessingStatus() == ProcessingStatus.PARSING) {
            throw new BatfishException(String.format("Cannot queue parsing work for %s / %s: " + "Status is PARSING but no incomplete parsing work exists", wDetails.baseTestrig, wDetails.baseEnv));
        } else if (envMetadata.getProcessingStatus() == ProcessingStatus.DATAPLANING) {
            throw new BatfishException(String.format("Cannot queue parsing work for %s / %s: " + "Status is DATAPLANING but no incomplete dataplaning work exists", wDetails.baseTestrig, wDetails.baseEnv));
        }
    }
    return _queueIncompleteWork.enque(work);
}
Also used : BatfishException(org.batfish.common.BatfishException) WorkItem(org.batfish.common.WorkItem) EnvironmentMetadata(org.batfish.datamodel.EnvironmentMetadata)

Example 32 with WorkItem

use of org.batfish.common.WorkItem in project batfish by batfish.

the class WorkQueueMgr method processTaskCheckResult.

public synchronized void processTaskCheckResult(QueuedWork work, Task task) throws Exception {
    switch(task.getStatus()) {
        case Unscheduled:
        case InProgress:
            work.setStatus(WorkStatusCode.ASSIGNED);
            work.recordTaskCheckResult(task);
            break;
        case TerminatedAbnormally:
        case TerminatedByUser:
        case TerminatedNormally:
        case RequeueFailure:
            {
                // move the work to completed queue
                _queueIncompleteWork.delete(work);
                _queueCompletedWork.enque(work);
                work.setStatus(WorkStatusCode.fromTerminatedTaskStatus(task.getStatus()));
                work.recordTaskCheckResult(task);
                // update testrig metadata
                WorkItem wItem = work.getWorkItem();
                WorkDetails wDetails = work.getDetails();
                if (wDetails.workType == WorkType.PARSING) {
                    ProcessingStatus status = (task.getStatus() == TaskStatus.TerminatedNormally) ? ProcessingStatus.PARSED : ProcessingStatus.PARSING_FAIL;
                    TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, status, task.getErrMessage());
                } else if (wDetails.workType == WorkType.DATAPLANING) {
                    // no change in status needed if task.getStatus() is RequeueFailure
                    if (task.getStatus() == TaskStatus.TerminatedAbnormally || task.getStatus() == TaskStatus.TerminatedByUser) {
                        TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.DATAPLANING_FAIL, task.getErrMessage());
                    } else if (task.getStatus() == TaskStatus.TerminatedNormally) {
                        TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.DATAPLANED, null);
                    }
                }
                // check if we unblocked anything
                if (_blockingWork.contains(wItem.getId())) {
                    _blockingWork.remove(wItem.getId());
                    List<QueuedWork> requeueWorks = new LinkedList<>();
                    for (QueuedWork incompleteWork : _queueIncompleteWork) {
                        if (incompleteWork.getStatus() == WorkStatusCode.BLOCKED && wDetails.isOverlappingInput(incompleteWork.getDetails())) {
                            requeueWorks.add(incompleteWork);
                        }
                    }
                    for (QueuedWork requeueWork : requeueWorks) {
                        _queueIncompleteWork.delete(requeueWork);
                        requeueWork.setStatus(WorkStatusCode.UNASSIGNED);
                    }
                    for (QueuedWork requeueWork : requeueWorks) {
                        try {
                            boolean queued = queueUnassignedWork(requeueWork);
                            if (!queued) {
                                throw new BatfishException("Failed to requeue previously blocked work " + requeueWork.getId());
                            }
                        } catch (Exception e) {
                            String stackTrace = ExceptionUtils.getStackTrace(e);
                            _logger.errorf("exception: %s\n", stackTrace);
                            // put this work back on incomplete queue and process as if it terminatedabnormally
                            // people may be checking its status and this work may be blocking others
                            _queueIncompleteWork.enque(requeueWork);
                            Task fakeTask = new Task(TaskStatus.RequeueFailure, "Couldn't requeue after unblocking");
                            processTaskCheckResult(requeueWork, fakeTask);
                        }
                    }
                }
            }
            break;
        case Unknown:
            // we mark this unassigned, so we try to schedule it again
            work.setStatus(WorkStatusCode.UNASSIGNED);
            work.clearAssignment();
            break;
        case UnreachableOrBadResponse:
            {
                if (work.getLastTaskCheckResult().getStatus() == TaskStatus.UnreachableOrBadResponse) {
                    // if we saw the same thing last time around, free the task to be scheduled elsewhere
                    work.setStatus(WorkStatusCode.UNASSIGNED);
                    work.clearAssignment();
                    work.recordTaskCheckResult(task);
                    // update testrig metadata
                    WorkItem wItem = work.getWorkItem();
                    WorkDetails wDetails = work.getDetails();
                    if (wDetails.workType == WorkType.PARSING || wDetails.workType == WorkType.DATAPLANING) {
                        EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv);
                        if (wDetails.workType == WorkType.PARSING) {
                            if (envMetadata.getProcessingStatus() != ProcessingStatus.PARSING) {
                                _logger.errorf("Unexpected status %s when parsing failed for %s / %s", envMetadata.getProcessingStatus(), wDetails.baseTestrig, wDetails.baseEnv);
                            } else {
                                TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.UNINITIALIZED, task.getErrMessage());
                            }
                        } else {
                            // wDetails.workType == WorkType.DATAPLANING
                            if (envMetadata.getProcessingStatus() != ProcessingStatus.DATAPLANING) {
                                _logger.errorf("Unexpected status %s when dataplaning failed for %s / %s", envMetadata.getProcessingStatus(), wDetails.baseTestrig, wDetails.baseEnv);
                            } else {
                                TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.PARSED, task.getErrMessage());
                            }
                        }
                    }
                } else {
                    work.setStatus(WorkStatusCode.ASSIGNED);
                    work.recordTaskCheckResult(task);
                }
            }
            break;
        default:
            throw new BatfishException("Unhandled " + TaskStatus.class.getCanonicalName() + ": " + task.getStatus());
    }
}
Also used : BatfishException(org.batfish.common.BatfishException) Task(org.batfish.common.Task) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List) WorkItem(org.batfish.common.WorkItem) ProcessingStatus(org.batfish.datamodel.EnvironmentMetadata.ProcessingStatus) EnvironmentMetadata(org.batfish.datamodel.EnvironmentMetadata) BatfishException(org.batfish.common.BatfishException) IOException(java.io.IOException) JSONException(org.codehaus.jettison.json.JSONException)

Example 33 with WorkItem

use of org.batfish.common.WorkItem in project batfish by batfish.

the class WorkQueueMgr method queueDataplaningWork.

private boolean queueDataplaningWork(QueuedWork work) throws Exception {
    WorkItem wItem = work.getWorkItem();
    WorkDetails wDetails = work.getDetails();
    QueuedWork currentDataplaningWork = getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, WorkType.DATAPLANING);
    if (currentDataplaningWork != null) {
        throw new BatfishException("Dataplaning is already in queue/progress");
    }
    // see comment in queueParsingWork for justification
    QueuedWork ddWork = getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, WorkType.DATAPLANE_DEPENDENT_ANSWERING);
    if (ddWork != null) {
        throw new BatfishException("Cannot queue dataplaning work while other dependent work exists");
    }
    QueuedWork blocker = getBlockerForDataplaningWork(work);
    if (blocker == null) {
        return _queueIncompleteWork.enque(work);
    } else {
        return queueBlockedWork(work, blocker);
    }
}
Also used : BatfishException(org.batfish.common.BatfishException) WorkItem(org.batfish.common.WorkItem)

Example 34 with WorkItem

use of org.batfish.common.WorkItem in project batfish by batfish.

the class WorkQueueMgr method getBlockerForParsingDependentWork.

private QueuedWork getBlockerForParsingDependentWork(QueuedWork work, String testrig, String environment) throws IOException {
    WorkItem wItem = work.getWorkItem();
    EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), testrig, environment);
    QueuedWork parsingWork = getIncompleteWork(wItem.getContainerName(), testrig, environment, WorkType.PARSING);
    switch(envMetadata.getProcessingStatus()) {
        case UNINITIALIZED:
        case PARSING:
        case PARSING_FAIL:
            if (parsingWork == null) {
                throw new BatfishException(String.format("Cannot queue parsing dependent work for %s / %s: " + "Status is %s but no incomplete parsing work exists", testrig, environment, envMetadata.getProcessingStatus()));
            }
            return parsingWork;
        case PARSED:
        case DATAPLANING:
        case DATAPLANED:
        case DATAPLANING_FAIL:
            return parsingWork;
        default:
            throw new BatfishException("Unknown testrig processingStatus: " + envMetadata.getProcessingStatus());
    }
}
Also used : BatfishException(org.batfish.common.BatfishException) WorkItem(org.batfish.common.WorkItem) EnvironmentMetadata(org.batfish.datamodel.EnvironmentMetadata)

Example 35 with WorkItem

use of org.batfish.common.WorkItem in project batfish by batfish.

the class WorkQueueMgr method getBlockerForDataplaneDependentWork.

/**
 * This function has a side effect It will inject a dataplane generation work in the queue if none
 * exists
 */
private QueuedWork getBlockerForDataplaneDependentWork(QueuedWork work, String testrig, String environment) throws Exception {
    WorkItem wItem = work.getWorkItem();
    EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), testrig, environment);
    QueuedWork parsingWork = getIncompleteWork(wItem.getContainerName(), testrig, environment, WorkType.PARSING);
    QueuedWork dataplaningWork = getIncompleteWork(wItem.getContainerName(), testrig, environment, WorkType.DATAPLANING);
    switch(envMetadata.getProcessingStatus()) {
        case UNINITIALIZED:
        case PARSING_FAIL:
        case PARSING:
            if (parsingWork == null) {
                throw new BatfishException(String.format("Cannot queue dataplane dependent work for %s / %s: " + "Status is %s but no incomplete parsing work exists", testrig, environment, envMetadata.getProcessingStatus()));
            }
            return parsingWork;
        case PARSED:
            if (parsingWork != null) {
                return parsingWork;
            }
            if (dataplaningWork != null) {
                return dataplaningWork;
            }
            return generateAndQueueDataplaneWork(wItem.getContainerName(), testrig, environment);
        case DATAPLANING_FAIL:
        case DATAPLANING:
            if (dataplaningWork == null) {
                throw new BatfishException(String.format("Cannot queue dataplane dependent work for %s / %s: " + "Status is %s but no incomplete dataplaning work exists", testrig, environment, envMetadata.getProcessingStatus()));
            }
            return dataplaningWork;
        case DATAPLANED:
            if (parsingWork != null) {
                return parsingWork;
            }
            if (dataplaningWork != null) {
                return dataplaningWork;
            }
            return null;
        default:
            throw new BatfishException("Unknown environment processingStatus: " + envMetadata.getProcessingStatus());
    }
}
Also used : BatfishException(org.batfish.common.BatfishException) WorkItem(org.batfish.common.WorkItem) EnvironmentMetadata(org.batfish.datamodel.EnvironmentMetadata)

Aggregations

WorkItem (org.batfish.common.WorkItem)44 Test (org.junit.Test)15 BatfishException (org.batfish.common.BatfishException)14 IOException (java.io.IOException)6 EnvironmentMetadata (org.batfish.datamodel.EnvironmentMetadata)5 JSONObject (org.codehaus.jettison.json.JSONObject)4 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)3 FileNotFoundException (java.io.FileNotFoundException)3 Path (java.nio.file.Path)3 AccessControlException (java.security.AccessControlException)3 POST (javax.ws.rs.POST)3 Path (javax.ws.rs.Path)3 Produces (javax.ws.rs.Produces)3 FileExistsException (org.apache.commons.io.FileExistsException)3 JSONException (org.codehaus.jettison.json.JSONException)3 LinkedList (java.util.LinkedList)2 Task (org.batfish.common.Task)2 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 JsonNode (com.fasterxml.jackson.databind.JsonNode)1 ArrayList (java.util.ArrayList)1