Examples with EnvironmentMetadata org.batfish.datamodel.EnvironmentMetadata used on opensource projects

Example 1 with EnvironmentMetadata

use of org.batfish.datamodel.EnvironmentMetadata in project batfish by batfish.

the class WorkQueueMgrTest method initTestrigMetadata.

private void initTestrigMetadata(String container, String testrig, String environment, ProcessingStatus status) throws JsonProcessingException {
    Path metadataPath = WorkMgr.getpathTestrigMetadata(container, testrig);
    metadataPath.getParent().toFile().mkdirs();
    TestrigMetadata trMetadata = new TestrigMetadata(Instant.now(), environment);
    EnvironmentMetadata envMetadata = trMetadata.getEnvironments().get(environment);
    envMetadata.updateStatus(status, null);
    TestrigMetadataMgr.writeMetadata(trMetadata, metadataPath);
}

Example 2 with EnvironmentMetadata

use of org.batfish.datamodel.EnvironmentMetadata in project batfish by batfish.

the class WorkQueueMgr method queueParsingWork.

private synchronized boolean queueParsingWork(QueuedWork work) throws Exception {
    WorkItem wItem = work.getWorkItem();
    WorkDetails wDetails = work.getDetails();
    // if incomplete work for this testrig exists, lets just reject this parsing work.
    // parsing work cannot proceeed in parallel because it may overwrite files used by others.
    // instead of rejecting, we could have queued it as BLOCKED but we risk cycles of BLOCKED work
    // this should not be a common case anyway, so we aren't losing much by rejecting it
    QueuedWork incompleteWork = getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, null);
    if (incompleteWork != null) {
        throw new BatfishException("Cannot queue parsing work while other work is incomplete");
    } else {
        EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv);
        if (envMetadata.getProcessingStatus() == ProcessingStatus.PARSING) {
            throw new BatfishException(String.format("Cannot queue parsing work for %s / %s: " + "Status is PARSING but no incomplete parsing work exists", wDetails.baseTestrig, wDetails.baseEnv));
        } else if (envMetadata.getProcessingStatus() == ProcessingStatus.DATAPLANING) {
            throw new BatfishException(String.format("Cannot queue parsing work for %s / %s: " + "Status is DATAPLANING but no incomplete dataplaning work exists", wDetails.baseTestrig, wDetails.baseEnv));
        }
    }
    return _queueIncompleteWork.enque(work);
}

Example 3 with EnvironmentMetadata

use of org.batfish.datamodel.EnvironmentMetadata in project batfish by batfish.

the class WorkQueueMgr method processTaskCheckResult.

public synchronized void processTaskCheckResult(QueuedWork work, Task task) throws Exception {
    switch(task.getStatus()) {
        case Unscheduled:
        case InProgress:
            work.setStatus(WorkStatusCode.ASSIGNED);
            work.recordTaskCheckResult(task);
            break;
        case TerminatedAbnormally:
        case TerminatedByUser:
        case TerminatedNormally:
        case RequeueFailure:
            {
                // move the work to completed queue
                _queueIncompleteWork.delete(work);
                _queueCompletedWork.enque(work);
                work.setStatus(WorkStatusCode.fromTerminatedTaskStatus(task.getStatus()));
                work.recordTaskCheckResult(task);
                // update testrig metadata
                WorkItem wItem = work.getWorkItem();
                WorkDetails wDetails = work.getDetails();
                if (wDetails.workType == WorkType.PARSING) {
                    ProcessingStatus status = (task.getStatus() == TaskStatus.TerminatedNormally) ? ProcessingStatus.PARSED : ProcessingStatus.PARSING_FAIL;
                    TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, status, task.getErrMessage());
                } else if (wDetails.workType == WorkType.DATAPLANING) {
                    // no change in status needed if task.getStatus() is RequeueFailure
                    if (task.getStatus() == TaskStatus.TerminatedAbnormally || task.getStatus() == TaskStatus.TerminatedByUser) {
                        TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.DATAPLANING_FAIL, task.getErrMessage());
                    } else if (task.getStatus() == TaskStatus.TerminatedNormally) {
                        TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.DATAPLANED, null);
                    }
                }
                // check if we unblocked anything
                if (_blockingWork.contains(wItem.getId())) {
                    _blockingWork.remove(wItem.getId());
                    List<QueuedWork> requeueWorks = new LinkedList<>();
                    for (QueuedWork incompleteWork : _queueIncompleteWork) {
                        if (incompleteWork.getStatus() == WorkStatusCode.BLOCKED && wDetails.isOverlappingInput(incompleteWork.getDetails())) {
                            requeueWorks.add(incompleteWork);
                        }
                    }
                    for (QueuedWork requeueWork : requeueWorks) {
                        _queueIncompleteWork.delete(requeueWork);
                        requeueWork.setStatus(WorkStatusCode.UNASSIGNED);
                    }
                    for (QueuedWork requeueWork : requeueWorks) {
                        try {
                            boolean queued = queueUnassignedWork(requeueWork);
                            if (!queued) {
                                throw new BatfishException("Failed to requeue previously blocked work " + requeueWork.getId());
                            }
                        } catch (Exception e) {
                            String stackTrace = ExceptionUtils.getStackTrace(e);
                            _logger.errorf("exception: %s\n", stackTrace);
                            // put this work back on incomplete queue and process as if it terminatedabnormally
                            // people may be checking its status and this work may be blocking others
                            _queueIncompleteWork.enque(requeueWork);
                            Task fakeTask = new Task(TaskStatus.RequeueFailure, "Couldn't requeue after unblocking");
                            processTaskCheckResult(requeueWork, fakeTask);
                        }
                    }
                }
            }
            break;
        case Unknown:
            // we mark this unassigned, so we try to schedule it again
            work.setStatus(WorkStatusCode.UNASSIGNED);
            work.clearAssignment();
            break;
        case UnreachableOrBadResponse:
            {
                if (work.getLastTaskCheckResult().getStatus() == TaskStatus.UnreachableOrBadResponse) {
                    // if we saw the same thing last time around, free the task to be scheduled elsewhere
                    work.setStatus(WorkStatusCode.UNASSIGNED);
                    work.clearAssignment();
                    work.recordTaskCheckResult(task);
                    // update testrig metadata
                    WorkItem wItem = work.getWorkItem();
                    WorkDetails wDetails = work.getDetails();
                    if (wDetails.workType == WorkType.PARSING || wDetails.workType == WorkType.DATAPLANING) {
                        EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv);
                        if (wDetails.workType == WorkType.PARSING) {
                            if (envMetadata.getProcessingStatus() != ProcessingStatus.PARSING) {
                                _logger.errorf("Unexpected status %s when parsing failed for %s / %s", envMetadata.getProcessingStatus(), wDetails.baseTestrig, wDetails.baseEnv);
                            } else {
                                TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.UNINITIALIZED, task.getErrMessage());
                            }
                        } else {
                            // wDetails.workType == WorkType.DATAPLANING
                            if (envMetadata.getProcessingStatus() != ProcessingStatus.DATAPLANING) {
                                _logger.errorf("Unexpected status %s when dataplaning failed for %s / %s", envMetadata.getProcessingStatus(), wDetails.baseTestrig, wDetails.baseEnv);
                            } else {
                                TestrigMetadataMgr.updateEnvironmentStatus(wItem.getContainerName(), wDetails.baseTestrig, wDetails.baseEnv, ProcessingStatus.PARSED, task.getErrMessage());
                            }
                        }
                    }
                } else {
                    work.setStatus(WorkStatusCode.ASSIGNED);
                    work.recordTaskCheckResult(task);
                }
            }
            break;
        default:
            throw new BatfishException("Unhandled " + TaskStatus.class.getCanonicalName() + ": " + task.getStatus());
    }
}

Example 4 with EnvironmentMetadata

use of org.batfish.datamodel.EnvironmentMetadata in project batfish by batfish.

the class WorkQueueMgr method getBlockerForParsingDependentWork.

private QueuedWork getBlockerForParsingDependentWork(QueuedWork work, String testrig, String environment) throws IOException {
    WorkItem wItem = work.getWorkItem();
    EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), testrig, environment);
    QueuedWork parsingWork = getIncompleteWork(wItem.getContainerName(), testrig, environment, WorkType.PARSING);
    switch(envMetadata.getProcessingStatus()) {
        case UNINITIALIZED:
        case PARSING:
        case PARSING_FAIL:
            if (parsingWork == null) {
                throw new BatfishException(String.format("Cannot queue parsing dependent work for %s / %s: " + "Status is %s but no incomplete parsing work exists", testrig, environment, envMetadata.getProcessingStatus()));
            }
            return parsingWork;
        case PARSED:
        case DATAPLANING:
        case DATAPLANED:
        case DATAPLANING_FAIL:
            return parsingWork;
        default:
            throw new BatfishException("Unknown testrig processingStatus: " + envMetadata.getProcessingStatus());
    }
}

Example 5 with EnvironmentMetadata

use of org.batfish.datamodel.EnvironmentMetadata in project batfish by batfish.

the class WorkQueueMgr method getBlockerForDataplaneDependentWork.

/**
 * This function has a side effect It will inject a dataplane generation work in the queue if none
 * exists
 */
private QueuedWork getBlockerForDataplaneDependentWork(QueuedWork work, String testrig, String environment) throws Exception {
    WorkItem wItem = work.getWorkItem();
    EnvironmentMetadata envMetadata = TestrigMetadataMgr.getEnvironmentMetadata(wItem.getContainerName(), testrig, environment);
    QueuedWork parsingWork = getIncompleteWork(wItem.getContainerName(), testrig, environment, WorkType.PARSING);
    QueuedWork dataplaningWork = getIncompleteWork(wItem.getContainerName(), testrig, environment, WorkType.DATAPLANING);
    switch(envMetadata.getProcessingStatus()) {
        case UNINITIALIZED:
        case PARSING_FAIL:
        case PARSING:
            if (parsingWork == null) {
                throw new BatfishException(String.format("Cannot queue dataplane dependent work for %s / %s: " + "Status is %s but no incomplete parsing work exists", testrig, environment, envMetadata.getProcessingStatus()));
            }
            return parsingWork;
        case PARSED:
            if (parsingWork != null) {
                return parsingWork;
            }
            if (dataplaningWork != null) {
                return dataplaningWork;
            }
            return generateAndQueueDataplaneWork(wItem.getContainerName(), testrig, environment);
        case DATAPLANING_FAIL:
        case DATAPLANING:
            if (dataplaningWork == null) {
                throw new BatfishException(String.format("Cannot queue dataplane dependent work for %s / %s: " + "Status is %s but no incomplete dataplaning work exists", testrig, environment, envMetadata.getProcessingStatus()));
            }
            return dataplaningWork;
        case DATAPLANED:
            if (parsingWork != null) {
                return parsingWork;
            }
            if (dataplaningWork != null) {
                return dataplaningWork;
            }
            return null;
        default:
            throw new BatfishException("Unknown environment processingStatus: " + envMetadata.getProcessingStatus());
    }
}