use of edu.pitt.dbmi.tetrad.db.entity.HpcJobInfo in project tetrad by cmu-phil.
the class HpcJobsScheduledTask method run.
// Pooling job status from HPC nodes
@Override
public void run() {
TetradDesktop desktop = (TetradDesktop) DesktopController.getInstance();
if (desktop == null)
return;
final HpcAccountManager hpcAccountManager = desktop.getHpcAccountManager();
// No Hpc Account in the first place, no need to proceed!
List<HpcAccount> hpcAccounts = hpcAccountManager.getHpcAccounts();
if (hpcAccounts == null || hpcAccounts.isEmpty())
return;
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
// LOGGER.debug("HpcJobsScheduledTask: " + new Date(System.currentTimeMillis()));
// Load active jobs: Status (0 = Submitted; 1 = Running; 2 = Kill
// Request)
Map<HpcAccount, Set<HpcJobInfo>> submittedHpcJobInfos = hpcJobManager.getSubmittedHpcJobInfoMap();
for (HpcAccount hpcAccount : submittedHpcJobInfos.keySet()) {
LOGGER.debug("HpcJobsScheduledTask: " + hpcAccount.getConnectionName());
Set<HpcJobInfo> hpcJobInfos = submittedHpcJobInfos.get(hpcAccount);
// Pid-HpcJobInfo map
Map<Long, HpcJobInfo> hpcJobInfoMap = new HashMap<>();
for (HpcJobInfo hpcJobInfo : hpcJobInfos) {
if (hpcJobInfo.getPid() != null) {
long pid = hpcJobInfo.getPid().longValue();
hpcJobInfoMap.put(pid, hpcJobInfo);
LOGGER.debug("id: " + hpcJobInfo.getId() + " : " + hpcJobInfo.getAlgoId() + ": pid: " + pid + " : " + hpcJobInfo.getResultFileName());
} else {
LOGGER.debug("id: " + hpcJobInfo.getId() + " : " + hpcJobInfo.getAlgoId() + ": no pid! : " + hpcJobInfo.getResultFileName());
hpcJobInfos.remove(hpcJobInfo);
}
}
// Finished job map
HashMap<Long, HpcJobInfo> finishedJobMap = new HashMap<>();
for (HpcJobInfo job : hpcJobInfos) {
finishedJobMap.put(job.getPid(), job);
}
try {
List<JobInfo> jobInfos = hpcJobManager.getRemoteActiveJobs(hpcAccountManager, hpcAccount);
for (JobInfo jobInfo : jobInfos) {
LOGGER.debug("Remote pid: " + jobInfo.getId() + " : " + jobInfo.getAlgoId() + " : " + jobInfo.getResultFileName());
long pid = jobInfo.getId();
if (finishedJobMap.containsKey(pid)) {
finishedJobMap.remove(pid);
}
int remoteStatus = jobInfo.getStatus();
String recentStatusText = (remoteStatus == 0 ? "Submitted" : (remoteStatus == 1 ? "Running" : "Kill Request"));
// Local job
HpcJobInfo hpcJobInfo = hpcJobInfoMap.get(pid);
// map
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
if (hpcJobInfo != null) {
int status = hpcJobInfo.getStatus();
if (status != remoteStatus) {
// Update status
hpcJobInfo.setStatus(remoteStatus);
hpcJobManager.updateHpcJobInfo(hpcJobInfo);
hpcJobLog.setLastUpdatedTime(new Date(System.currentTimeMillis()));
String log = "Job status changed to " + recentStatusText;
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : pid : " + pid);
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, remoteStatus, log);
}
}
}
// Download finished jobs' results
if (finishedJobMap.size() > 0) {
Set<ResultFile> resultFiles = hpcJobManager.listRemoteAlgorithmResultFiles(hpcAccountManager, hpcAccount);
Set<String> resultFileNames = new HashSet<>();
for (ResultFile resultFile : resultFiles) {
resultFileNames.add(resultFile.getName());
// LOGGER.debug(hpcAccount.getConnectionName()
// + " Result : " + resultFile.getName());
}
for (HpcJobInfo hpcJobInfo : finishedJobMap.values()) {
// Job
// is
// done
// or
// killed or
// time-out
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
String recentStatusText = "Job finished";
// Finished
int recentStatus = 3;
if (hpcJobInfo.getStatus() == 2) {
recentStatusText = "Job killed";
// Killed
recentStatus = 4;
}
hpcJobInfo.setStatus(recentStatus);
hpcJobManager.updateHpcJobInfo(hpcJobInfo);
// LOGGER.debug("hpcJobInfo: id: "
// + hpcJobInfo.getId() + " : "
// + hpcJobInfo.getStatus());
hpcJobManager.logHpcJobLogDetail(hpcJobLog, recentStatus, recentStatusText);
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : " + recentStatusText);
GeneralAlgorithmEditor editor = hpcJobManager.getGeneralAlgorithmEditor(hpcJobInfo);
if (editor != null) {
LOGGER.debug("GeneralAlgorithmEditor is not null");
String resultJsonFileName = hpcJobInfo.getResultJsonFileName();
String errorResultFileName = hpcJobInfo.getErrorResultFileName();
if (resultFileNames.contains(resultJsonFileName)) {
// Result Downloaded
recentStatus = 5;
String json = downloadAlgorithmResultFile(hpcAccountManager, hpcJobManager, hpcAccount, resultJsonFileName, editor);
if (!json.toLowerCase().contains("not found")) {
editor.setAlgorithmResult(json);
}
String log = "Result downloaded";
hpcJobManager.logHpcJobLogDetail(hpcJobLog, recentStatus, log);
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : " + log);
} else if (resultFileNames.contains(errorResultFileName)) {
// Error Result Downloaded
recentStatus = 6;
String error = downloadAlgorithmResultFile(hpcAccountManager, hpcJobManager, hpcAccount, errorResultFileName, editor);
if (!error.toLowerCase().contains("not found")) {
editor.setAlgorithmErrorResult(error);
}
String log = "Error Result downloaded";
hpcJobManager.logHpcJobLogDetail(hpcJobLog, recentStatus, log);
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : " + log);
} else {
// Try again
Thread.sleep(5000);
String json = downloadAlgorithmResultFile(hpcAccountManager, hpcJobManager, hpcAccount, resultJsonFileName, editor);
if (!json.toLowerCase().contains("not found")) {
editor.setAlgorithmResult(json);
// Result Downloaded
recentStatus = 5;
String log = "Result downloaded";
hpcJobManager.logHpcJobLogDetail(hpcJobLog, recentStatus, log);
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : " + log);
} else {
String error = downloadAlgorithmResultFile(hpcAccountManager, hpcJobManager, hpcAccount, errorResultFileName, editor);
if (!error.toLowerCase().contains("not found")) {
editor.setAlgorithmErrorResult(error);
// Error Result
recentStatus = 6;
// Downloaded
String log = "Error Result downloaded";
hpcJobManager.logHpcJobLogDetail(hpcJobLog, recentStatus, log);
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : " + log);
} else {
// Result Not Found
recentStatus = 7;
String log = resultJsonFileName + " not found";
hpcJobManager.logHpcJobLogDetail(hpcJobLog, recentStatus, log);
LOGGER.debug(hpcJobInfo.getAlgoId() + " : id : " + hpcJobInfo.getId() + " : " + log);
}
}
}
}
hpcJobManager.removeFinishedHpcJob(hpcJobInfo);
}
} else {
LOGGER.debug("No finished job yet.");
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
use of edu.pitt.dbmi.tetrad.db.entity.HpcJobInfo in project tetrad by cmu-phil.
the class DeleteHpcJobInfoAction method actionPerformed.
@Override
public void actionPerformed(ActionEvent e) {
JTable table = (JTable) e.getSource();
int modelRow = Integer.valueOf(e.getActionCommand());
DefaultTableModel finishedJobTableModel = (DefaultTableModel) table.getModel();
long jobId = Long.valueOf(finishedJobTableModel.getValueAt(modelRow, HpcJobActivityEditor.ID_COLUMN).toString()).longValue();
int answer = JOptionPane.showConfirmDialog(parentComp, "Would you like to delete this HPC job id: " + jobId + "?", "Delete HPC job", JOptionPane.YES_NO_OPTION);
if (answer == JOptionPane.NO_OPTION)
return;
TetradDesktop desktop = (TetradDesktop) DesktopController.getInstance();
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
HpcJobInfo hpcJobInfo = hpcJobManager.findHpcJobInfoById(Long.valueOf(finishedJobTableModel.getValueAt(modelRow, HpcJobActivityEditor.ID_COLUMN).toString()).longValue());
if (hpcJobInfo != null) {
// Update table
finishedJobTableModel.removeRow(modelRow);
table.updateUI();
hpcJobManager.removeHpcJobInfoTransaction(hpcJobInfo);
}
}
use of edu.pitt.dbmi.tetrad.db.entity.HpcJobInfo in project tetrad by cmu-phil.
the class HpcJobActivityEditor method getActiveRowData.
private Vector<Vector<String>> getActiveRowData(final TetradDesktop desktop, final List<HpcAccount> exclusiveHpcAccounts) throws Exception {
final Vector<Vector<String>> activeRowData = new Vector<>();
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
Map<Long, HpcJobInfo> activeHpcJobInfoMap = null;
// Pending
Map<HpcAccount, Set<HpcJobInfo>> pendingHpcJobInfoMap = hpcJobManager.getPendingHpcJobInfoMap();
pendingDisplayHpcJobInfoSet.clear();
for (HpcAccount hpcAccount : pendingHpcJobInfoMap.keySet()) {
if (exclusiveHpcAccounts != null && !exclusiveHpcAccounts.contains(hpcAccount)) {
continue;
}
Set<HpcJobInfo> pendingHpcJobSet = pendingHpcJobInfoMap.get(hpcAccount);
for (HpcJobInfo hpcJobInfo : pendingHpcJobSet) {
// For monitoring purpose
pendingDisplayHpcJobInfoSet.add(hpcJobInfo);
if (activeHpcJobInfoMap == null) {
activeHpcJobInfoMap = new HashMap<>();
}
activeHpcJobInfoMap.put(hpcJobInfo.getId(), hpcJobInfo);
}
}
// Submitted
Map<HpcAccount, Set<HpcJobInfo>> submittedHpcJobInfoMap = hpcJobManager.getSubmittedHpcJobInfoMap();
submittedDisplayHpcJobInfoSet.clear();
for (HpcAccount hpcAccount : submittedHpcJobInfoMap.keySet()) {
if (exclusiveHpcAccounts != null && !exclusiveHpcAccounts.contains(hpcAccount)) {
continue;
}
Set<HpcJobInfo> submittedHpcJobSet = submittedHpcJobInfoMap.get(hpcAccount);
for (HpcJobInfo hpcJobInfo : submittedHpcJobSet) {
// For monitoring purpose
submittedDisplayHpcJobInfoSet.add(hpcJobInfo);
if (activeHpcJobInfoMap == null) {
activeHpcJobInfoMap = new HashMap<>();
}
activeHpcJobInfoMap.put(hpcJobInfo.getId(), hpcJobInfo);
}
}
if (activeHpcJobInfoMap != null) {
List<Long> activeJobIds = new ArrayList<>(activeHpcJobInfoMap.keySet());
Collections.sort(activeJobIds);
Collections.reverse(activeJobIds);
for (Long jobId : activeJobIds) {
final HpcJobInfo hpcJobInfo = activeHpcJobInfoMap.get(jobId);
Vector<String> rowData = new Vector<>();
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
// Local job id
rowData.add(hpcJobInfo.getId().toString());
int status = hpcJobInfo.getStatus();
switch(status) {
case -1:
rowData.add("Pending");
break;
case 0:
rowData.add("Submitted");
break;
case 1:
rowData.add("Running");
break;
case 2:
rowData.add("Kill Request");
break;
}
// Locally added time
rowData.add(FilePrint.fileTimestamp(hpcJobLog.getAddedTime().getTime()));
// HPC node name
HpcAccount hpcAccount = hpcJobInfo.getHpcAccount();
rowData.add(hpcAccount.getConnectionName());
// Algorithm
rowData.add(hpcJobInfo.getAlgoId());
// Dataset uploading progress
AlgorithmParamRequest algorParamReq = hpcJobInfo.getAlgorithmParamRequest();
String datasetPath = algorParamReq.getDatasetPath();
int progress = hpcJobManager.getUploadFileProgress(datasetPath);
if (progress > -1 && progress < 100) {
rowData.add("" + progress + "%");
} else {
rowData.add("Done");
}
// Prior Knowledge uploading progress
String priorKnowledgePath = algorParamReq.getPriorKnowledgePath();
if (priorKnowledgePath != null) {
progress = hpcJobManager.getUploadFileProgress(priorKnowledgePath);
if (progress > -1 && progress < 100) {
rowData.add("" + progress + "%");
} else {
rowData.add("Done");
}
} else {
rowData.add("Skipped");
}
if (status > -1) {
// Submitted time
rowData.add(FilePrint.fileTimestamp(hpcJobInfo.getSubmittedTime().getTime()));
// HPC job id
rowData.add(hpcJobInfo.getPid() != null ? "" + hpcJobInfo.getPid() : "");
} else {
rowData.add("");
rowData.add("");
}
// Last update time
rowData.add(FilePrint.fileTimestamp(hpcJobLog.getLastUpdatedTime().getTime()));
// Cancel job
rowData.add("Cancel");
activeRowData.add(rowData);
}
}
return activeRowData;
}
use of edu.pitt.dbmi.tetrad.db.entity.HpcJobInfo in project tetrad by cmu-phil.
the class HpcJobManager method submitNewHpcJobToQueue.
public synchronized void submitNewHpcJobToQueue(final HpcJobInfo hpcJobInfo, final GeneralAlgorithmEditor generalAlgorithmEditor) {
hpcJobInfoService.add(hpcJobInfo);
LOGGER.debug("hpcJobInfo: id: " + hpcJobInfo.getId());
HpcJobLog hpcJobLog = new HpcJobLog();
hpcJobLog.setAddedTime(new Date(System.currentTimeMillis()));
hpcJobLog.setHpcJobInfo(hpcJobInfo);
hpcJobLogService.update(hpcJobLog);
LOGGER.debug("HpcJobLog: id: " + hpcJobLog.getId());
HpcJobLogDetail hpcJobLogDetail = new HpcJobLogDetail();
hpcJobLogDetail.setAddedTime(new Date());
hpcJobLogDetail.setHpcJobLog(hpcJobLog);
// Pending
hpcJobLogDetail.setJobState(-1);
hpcJobLogDetail.setProgress("Pending");
hpcJobLogDetailService.add(hpcJobLogDetail);
LOGGER.debug("HpcJobLogDetail: id: " + hpcJobLogDetail.getId());
hpcGraphResultMap.put(hpcJobInfo, generalAlgorithmEditor);
// Put a new pre-process task into hpc job queue
HpcJobPreProcessTask preProcessTask = new HpcJobPreProcessTask(hpcJobInfo);
// Added a job to the pending list
final HpcAccount hpcAccount = hpcJobInfo.getHpcAccount();
Set<HpcJobInfo> hpcJobInfos = pendingHpcJobInfoMap.get(hpcAccount);
if (hpcJobInfos == null) {
hpcJobInfos = new LinkedHashSet<>();
}
hpcJobInfos.add(hpcJobInfo);
pendingHpcJobInfoMap.put(hpcAccount, hpcJobInfos);
executorService.execute(preProcessTask);
}
use of edu.pitt.dbmi.tetrad.db.entity.HpcJobInfo in project tetrad by cmu-phil.
the class HpcJobManager method requestHpcJobKilled.
public HpcJobInfo requestHpcJobKilled(final HpcJobInfo hpcJobInfo) throws Exception {
final HpcAccount hpcAccount = hpcJobInfo.getHpcAccount();
HpcAccountService hpcAccountService = getHpcAccountService(hpcAccount);
JobQueueService jobQueueService = hpcAccountService.getJobQueueService();
TetradDesktop desktop = (TetradDesktop) DesktopController.getInstance();
final HpcAccountManager hpcAccountManager = desktop.getHpcAccountManager();
JsonWebTokenManager jsonWebTokenManager = hpcAccountManager.getJsonWebTokenManager();
jobQueueService.requestJobKilled(hpcJobInfo.getPid(), jsonWebTokenManager.getJsonWebToken(hpcAccount));
JobInfo jobInfo = jobQueueService.getJobStatus(hpcJobInfo.getPid(), jsonWebTokenManager.getJsonWebToken(hpcAccount));
if (jobInfo != null) {
hpcJobInfo.setStatus(jobInfo.getStatus());
return hpcJobInfo;
}
return null;
}
Aggregations