Examples with RunningStateInfo cbit.vcell.server.RunningStateInfo used on opensource projects

Example 1 with RunningStateInfo

use of cbit.vcell.server.RunningStateInfo in project vcell by virtualcell.

the class SimulationStateMachine method onWorkerEvent.

public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException {
    updateSolverProcessTimestamp();
    WorkerEventMessage workerEventMessage = new WorkerEventMessage(workerEvent);
    VCMongoMessage.sendWorkerEvent(workerEventMessage);
    // as the filter of the client
    String userName = workerEvent.getUserName();
    int workerEventTaskID = workerEvent.getTaskID();
    if (lg.isTraceEnabled())
        lg.trace("onWorkerEventMessage[" + workerEvent.getEventTypeID() + "," + workerEvent.getSimulationMessage() + "][simid=" + workerEvent.getVCSimulationDataIdentifier() + ",job=" + jobIndex + ",task=" + workerEventTaskID + "]");
    VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier();
    if (vcSimDataID == null) {
        VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent - no SimID in message): " + workerEvent.show());
        return;
    }
    KeyValue simKey = vcSimDataID.getSimulationKey();
    SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex);
    if (oldSimulationJobStatus == null) {
        VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: " + workerEvent.show());
        return;
    }
    if (oldSimulationJobStatus == null || oldSimulationJobStatus.getSchedulerStatus().isDone() || oldSimulationJobStatus.getTaskID() > workerEventTaskID) {
        VCMongoMessage.sendInfo("onWorkerEvent() ignoring outdated WorkerEvent, (currState=" + oldSimulationJobStatus.getSchedulerStatus().getDescription() + "): " + workerEvent.show());
        return;
    }
    int taskID = oldSimulationJobStatus.getTaskID();
    SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus();
    // 
    // status information (initialized as if new record)
    // 
    Date startDate = null;
    Date lastUpdateDate = null;
    Date endDate = null;
    boolean hasData = false;
    HtcJobID htcJobID = null;
    String computeHost = null;
    VCellServerID vcServerID = VCellServerID.getSystemServerID();
    Date submitDate = null;
    Date queueDate = null;
    int queuePriority = PRIORITY_DEFAULT;
    SimulationJobStatus.SimulationQueueID simQueueID = SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING;
    // 
    // update using previously stored status (if available).
    // 
    SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus();
    if (oldSimExeStatus != null && oldSimExeStatus.getStartDate() != null) {
        startDate = oldSimExeStatus.getStartDate();
    }
    if (oldSimExeStatus != null && oldSimExeStatus.getLatestUpdateDate() != null) {
        lastUpdateDate = oldSimExeStatus.getLatestUpdateDate();
    }
    if (oldSimExeStatus != null && oldSimExeStatus.getEndDate() != null) {
        endDate = oldSimExeStatus.getEndDate();
    }
    if (oldSimExeStatus != null && oldSimExeStatus.hasData()) {
        hasData = true;
    }
    if (oldSimExeStatus != null && oldSimExeStatus.getComputeHost() != null) {
        computeHost = oldSimExeStatus.getComputeHost();
    }
    if (oldSimExeStatus != null && oldSimExeStatus.getHtcJobID() != null) {
        htcJobID = oldSimExeStatus.getHtcJobID();
    }
    vcServerID = oldSimulationJobStatus.getServerID();
    submitDate = oldSimulationJobStatus.getSubmitDate();
    SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus();
    if (oldQueueStatus != null && oldQueueStatus.getQueueDate() != null) {
        queueDate = oldQueueStatus.getQueueDate();
    }
    if (oldQueueStatus != null) {
        queuePriority = oldQueueStatus.getQueuePriority();
    }
    if (oldQueueStatus != null && oldQueueStatus.getQueueID() != null) {
        simQueueID = oldQueueStatus.getQueueID();
    }
    // 
    if (workerEvent.getHtcJobID() != null) {
        htcJobID = workerEvent.getHtcJobID();
    }
    if (workerEvent.getHostName() != null) {
        computeHost = workerEvent.getHostName();
    }
    SimulationMessage workerEventSimulationMessage = workerEvent.getSimulationMessage();
    if (workerEventSimulationMessage.getHtcJobId() != null) {
        htcJobID = workerEventSimulationMessage.getHtcJobId();
    }
    SimulationJobStatus newJobStatus = null;
    if (workerEvent.isAcceptedEvent()) {
        // 
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) {
            // new queue status
            SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
            // new exe status
            lastUpdateDate = new Date();
            startDate = lastUpdateDate;
            endDate = null;
            SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
            newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.DISPATCHED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
        }
    } else if (workerEvent.isStartingEvent()) {
        // only update database when the job event changes from started to running. The later progress event will not be recorded.
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) {
            // new queue status
            SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
            // new exe status
            lastUpdateDate = new Date();
            if (startDate == null) {
                startDate = lastUpdateDate;
            }
            SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
            newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
        }
    } else if (workerEvent.isNewDataEvent()) {
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) {
            if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData == false) {
                // new queue status
                SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
                // new exe status
                if (startDate == null) {
                    startDate = lastUpdateDate;
                }
                hasData = true;
                SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
                newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
            }
        }
    } else if (workerEvent.isProgressEvent() || workerEvent.isWorkerAliveEvent()) {
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) {
            if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL) {
                // new queue status
                SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
                // new exe status
                if (startDate == null) {
                    startDate = lastUpdateDate;
                }
                SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
                newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
            } else if (oldSchedulerStatus.isRunning()) {
                if (oldSimExeStatus != null) {
                    // Date latestUpdate = oldSimExeStatus.getLatestUpdateDate();
                    // if (System.currentTimeMillis() - latestUpdate.getTime() >= MessageConstants.INTERVAL_PING_SERVER_MS * 3 / 5) {
                    // new queue status
                    SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
                    SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
                    newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
                }
            // }
            }
        }
    } else if (workerEvent.isCompletedEvent()) {
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) {
            // new queue status
            SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
            // new exe status
            endDate = new Date();
            hasData = true;
            SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
            newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.COMPLETED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
        }
    } else if (workerEvent.isFailedEvent()) {
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) {
            // new queue status
            SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
            // new exe status
            endDate = new Date();
            SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
            newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.FAILED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus);
        }
    } else if (workerEvent.isWorkerExitErrorEvent()) {
        if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) {
            // new queue status
            SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL);
            // new exe status
            endDate = new Date();
            SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID);
            SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, " + workerEventSimulationMessage.getDisplayMessage());
            newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.FAILED, taskID, simulationMessage, newQueueStatus, newExeStatus);
        }
    }
    if (newJobStatus != null) {
        if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) {
            Double progress = workerEvent.getProgress();
            Double timepoint = workerEvent.getTimePoint();
            RunningStateInfo runningStateInfo = null;
            if (progress != null && timepoint != null) {
                runningStateInfo = new RunningStateInfo(progress, timepoint);
            }
            simulationDatabase.updateSimulationJobStatus(newJobStatus, runningStateInfo);
            StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint);
            msgForClient.sendToClient(session);
            if (lg.isTraceEnabled())
                lg.trace("Send status to client: " + msgForClient);
        } else {
            simulationDatabase.updateSimulationJobStatus(newJobStatus);
            StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null);
            msgForClient.sendToClient(session);
            if (lg.isTraceEnabled())
                lg.trace("Send status to client: " + msgForClient);
        }
    } else if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) {
        Double progress = workerEvent.getProgress();
        Double timepoint = workerEvent.getTimePoint();
        RunningStateInfo runningStateInfo = null;
        if (progress != null && timepoint != null) {
            runningStateInfo = new RunningStateInfo(progress, timepoint);
        }
        simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus, runningStateInfo);
        StatusMessage msgForClient = new StatusMessage(oldSimulationJobStatus, userName, progress, timepoint);
        msgForClient.sendToClient(session);
        if (lg.isTraceEnabled())
            lg.trace("Send status to client: " + msgForClient);
    } else {
        VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent (currState=" + oldSchedulerStatus.getDescription() + "): " + workerEvent.show());
    }
// addStateMachineTransition(new StateMachineTransition(new WorkerStateMachineEvent(taskID, workerEvent), oldSimulationJobStatus, newJobStatus));
}