use of edu.cmu.tetradapp.app.hpc.manager.HpcJobManager in project tetrad by cmu-phil.
the class HpcJobActivityEditor method getFinishedRowData.
private Vector<Vector<String>> getFinishedRowData(final TetradDesktop desktop, final List<HpcAccount> exclusiveHpcAccounts) throws Exception {
final Vector<Vector<String>> finishedRowData = new Vector<>();
HpcJobManager hpcJobManager = desktop.getHpcJobManager();
Map<Long, HpcJobInfo> finishedHpcJobIdMap = null;
// Finished jobs
Map<HpcAccount, Set<HpcJobInfo>> finishedHpcJobInfoMap = hpcJobManager.getFinishedHpcJobInfoMap();
for (HpcAccount hpcAccount : finishedHpcJobInfoMap.keySet()) {
if (exclusiveHpcAccounts != null && !exclusiveHpcAccounts.contains(hpcAccount)) {
continue;
}
Set<HpcJobInfo> finishedHpcJobSet = finishedHpcJobInfoMap.get(hpcAccount);
for (HpcJobInfo hpcJobInfo : finishedHpcJobSet) {
if (finishedHpcJobIdMap == null) {
finishedHpcJobIdMap = new HashMap<>();
}
finishedHpcJobIdMap.put(hpcJobInfo.getId(), hpcJobInfo);
}
}
if (finishedHpcJobIdMap != null) {
List<Long> finishedJobIds = new ArrayList<>(finishedHpcJobIdMap.keySet());
Collections.sort(finishedJobIds);
Collections.reverse(finishedJobIds);
for (Long jobId : finishedJobIds) {
final HpcJobInfo hpcJobInfo = finishedHpcJobIdMap.get(jobId);
Vector<String> rowData = new Vector<>();
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
// Local job id
rowData.add(hpcJobInfo.getId().toString());
int status = hpcJobInfo.getStatus();
switch(status) {
case 3:
rowData.add("Finished");
break;
case 4:
rowData.add("Canceled");
break;
case 5:
rowData.add("Finished");
break;
case 6:
rowData.add("Error");
break;
}
// Locally added time
rowData.add(FilePrint.fileTimestamp(hpcJobLog.getAddedTime().getTime()));
// HPC node name
HpcAccount hpcAccount = hpcJobInfo.getHpcAccount();
rowData.add(hpcAccount.getConnectionName());
// Algorithm
rowData.add(hpcJobInfo.getAlgoId());
// Submitted time
rowData.add(hpcJobInfo.getSubmittedTime() != null ? FilePrint.fileTimestamp(hpcJobInfo.getSubmittedTime().getTime()) : "");
// HPC job id
rowData.add("" + hpcJobInfo.getPid());
// Result Name
switch(status) {
case 3:
rowData.add(hpcJobInfo.getResultFileName());
break;
case 4:
rowData.add("");
break;
case 5:
rowData.add(hpcJobInfo.getResultFileName());
break;
case 6:
rowData.add(hpcJobInfo.getErrorResultFileName());
break;
}
// Finished time
if (status != 4) {
rowData.add(FilePrint.fileTimestamp(hpcJobLog.getEndedTime().getTime()));
} else {
rowData.add("");
}
// Canceled time
if (status == 4) {
rowData.add(hpcJobLog.getCanceledTime() != null ? FilePrint.fileTimestamp(hpcJobLog.getCanceledTime().getTime()) : "");
} else {
rowData.add("");
}
// Last update time
rowData.add(FilePrint.fileTimestamp(hpcJobLog.getLastUpdatedTime().getTime()));
// Delete job from db
rowData.add("Delete");
finishedRowData.add(rowData);
}
}
return finishedRowData;
}
use of edu.cmu.tetradapp.app.hpc.manager.HpcJobManager in project tetrad by cmu-phil.
the class HpcJobPreProcessTask method run.
@Override
public void run() {
TetradDesktop desktop = (TetradDesktop) DesktopController.getInstance();
while (desktop == null) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
final HpcAccountManager hpcAccountManager = desktop.getHpcAccountManager();
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
HpcAccount hpcAccount = hpcJobInfo.getHpcAccount();
AlgorithmParamRequest algorParamReq = hpcJobInfo.getAlgorithmParamRequest();
String datasetPath = algorParamReq.getDatasetPath();
String priorKnowledgePath = algorParamReq.getPriorKnowledgePath();
try {
HpcAccountService hpcAccountService = hpcJobManager.getHpcAccountService(hpcAccount);
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
String log = "Initiated connection to " + hpcAccount.getConnectionName();
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
log = "datasetPath: " + datasetPath;
System.out.println(log);
Path file = Paths.get(datasetPath);
// Get file's MD5 hash and use it as its identifier
String md5 = algorParamReq.getDatasetMd5();
// Initiate data uploading progress
hpcJobManager.updateUploadFileProgress(datasetPath, 0);
Path prior = null;
if (priorKnowledgePath != null) {
log = "priorKnowledgePath: " + priorKnowledgePath;
LOGGER.debug(log);
prior = Paths.get(priorKnowledgePath);
// Initiate prior knowledge uploading progress
hpcJobManager.updateUploadFileProgress(priorKnowledgePath, 0);
}
// Check if this dataset already exists with this md5 hash
RemoteDataFileService remoteDataService = hpcAccountService.getRemoteDataService();
DataFile dataFile = HpcAccountUtils.getRemoteDataFile(hpcAccountManager, remoteDataService, hpcAccount, md5);
DataUploadService dataUploadService = hpcAccountService.getDataUploadService();
// If not, upload the file
if (dataFile == null) {
log = "Started uploading " + file.getFileName().toString();
LOGGER.debug(log);
dataUploadService.startUpload(file, HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
int progress;
while ((progress = dataUploadService.getUploadJobStatus(file.toAbsolutePath().toString())) < 100) {
// System.out.println("Uploading "
// + file.toAbsolutePath().toString() + " Progress: "
// + progress + "%");
hpcJobManager.updateUploadFileProgress(datasetPath, progress);
Thread.sleep(10);
}
hpcJobManager.updateUploadFileProgress(datasetPath, progress);
log = "Finished uploading " + file.getFileName().toString();
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
// Get remote datafile
dataFile = HpcAccountUtils.getRemoteDataFile(hpcAccountManager, remoteDataService, hpcAccount, md5);
HpcAccountUtils.summarizeDataset(remoteDataService, algorParamReq, dataFile.getId(), HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
log = "Summarized " + file.getFileName().toString();
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
} else {
log = "Skipped uploading " + file.getFileName().toString();
LOGGER.debug(log);
hpcJobManager.updateUploadFileProgress(datasetPath, -1);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
if (dataFile.getFileSummary().getVariableType() == null) {
HpcAccountUtils.summarizeDataset(remoteDataService, algorParamReq, dataFile.getId(), HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
log = "Summarized " + file.getFileName().toString();
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, "Summarized " + file.getFileName().toString());
}
}
DataFile priorKnowledgeFile = null;
// Prior Knowledge File
if (prior != null) {
// Get prior knowledge file Id
md5 = algorParamReq.getPriorKnowledgeMd5();
priorKnowledgeFile = HpcAccountUtils.getRemotePriorKnowledgeFile(hpcAccountManager, remoteDataService, hpcAccount, md5);
if (priorKnowledgeFile == null) {
// Upload prior knowledge file
dataUploadService.startUpload(prior, HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
log = "Started uploading Prior Knowledge File";
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
int progress;
while ((progress = dataUploadService.getUploadJobStatus(prior.toAbsolutePath().toString())) < 100) {
hpcJobManager.updateUploadFileProgress(priorKnowledgePath, progress);
Thread.sleep(10);
}
hpcJobManager.updateUploadFileProgress(priorKnowledgePath, progress);
priorKnowledgeFile = HpcAccountUtils.getRemotePriorKnowledgeFile(hpcAccountManager, remoteDataService, hpcAccount, md5);
log = "Finished uploading Prior Knowledge File";
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, -1, log);
}
}
// Algorithm Job Preparation
edu.pitt.dbmi.ccd.rest.client.dto.algo.AlgorithmParamRequest paramRequest = new edu.pitt.dbmi.ccd.rest.client.dto.algo.AlgorithmParamRequest();
String algoId = hpcJobInfo.getAlgoId();
paramRequest.setAlgoId(algoId);
paramRequest.setDatasetFileId(dataFile.getId());
// Test
if (algorParamReq.getTestId() != null) {
paramRequest.setTestId(algorParamReq.getTestId());
}
// Score
if (algorParamReq.getScoreId() != null) {
paramRequest.setScoreId(algorParamReq.getScoreId());
}
Set<AlgoParameter> algorithmParameters = new HashSet<>();
for (AlgorithmParameter param : algorParamReq.getAlgorithmParameters()) {
algorithmParameters.add(new AlgoParameter(param.getParameter(), param.getValue()));
LOGGER.debug("AlgorithmParameter: " + param.getParameter() + " : " + param.getValue());
}
if (priorKnowledgeFile != null) {
paramRequest.setPriorKnowledgeFileId(priorKnowledgeFile.getId());
LOGGER.debug("priorKnowledgeFileId: " + priorKnowledgeFile.getId());
}
paramRequest.setAlgoParameters(algorithmParameters);
if (algorParamReq.getJvmOptions() != null) {
JvmOptions jvmOptions = new JvmOptions();
jvmOptions.setMaxHeapSize(algorParamReq.getJvmOptions().getMaxHeapSize());
paramRequest.setJvmOptions(jvmOptions);
}
Set<HpcParameter> hpcParameters = algorParamReq.getHpcParameters();
if (hpcParameters != null) {
Set<edu.pitt.dbmi.ccd.rest.client.dto.algo.HpcParameter> hpcParams = new HashSet<>();
for (HpcParameter param : hpcParameters) {
edu.pitt.dbmi.ccd.rest.client.dto.algo.HpcParameter hpcParam = new edu.pitt.dbmi.ccd.rest.client.dto.algo.HpcParameter();
hpcParam.setKey(param.getKey());
hpcParam.setValue(param.getValue());
hpcParams.add(hpcParam);
LOGGER.debug("HpcParameter: " + hpcParam.getKey() + " : " + hpcParam.getValue());
}
paramRequest.setHpcParameters(hpcParams);
}
// Submit a job
JobQueueService jobQueueService = hpcAccountService.getJobQueueService();
JobInfo jobInfo = jobQueueService.addToRemoteQueue(paramRequest, HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
// Log the job submission
hpcJobInfo.setSubmittedTime(new Date(System.currentTimeMillis()));
// Submitted
hpcJobInfo.setStatus(0);
hpcJobInfo.setPid(jobInfo.getId());
hpcJobInfo.setResultFileName(jobInfo.getResultFileName());
hpcJobInfo.setResultJsonFileName(jobInfo.getResultJsonFileName());
hpcJobInfo.setErrorResultFileName(jobInfo.getErrorResultFileName());
hpcJobManager.updateHpcJobInfo(hpcJobInfo);
log = "Submitted job to " + hpcAccount.getConnectionName();
LOGGER.debug(log);
hpcJobManager.logHpcJobLogDetail(hpcJobLog, 0, log);
LOGGER.debug("HpcJobPreProcessTask: HpcJobInfo: id : " + hpcJobInfo.getId() + " : pid : " + hpcJobInfo.getPid() + " : " + hpcJobInfo.getAlgoId() + hpcJobInfo.getAlgorithmParamRequest().getTestId() == null ? "" : " : " + hpcJobInfo.getAlgorithmParamRequest().getTestId() + hpcJobInfo.getAlgorithmParamRequest().getScoreId() == null ? "" : " : " + hpcJobInfo.getAlgorithmParamRequest().getScoreId() + " : " + hpcJobInfo.getResultFileName());
hpcJobManager.addNewSubmittedHpcJob(hpcJobInfo);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
use of edu.cmu.tetradapp.app.hpc.manager.HpcJobManager in project tetrad by cmu-phil.
the class KillHpcJobAction method actionPerformed.
@Override
public void actionPerformed(ActionEvent e) {
JTable table = (JTable) e.getSource();
int modelRow = Integer.valueOf(e.getActionCommand());
DefaultTableModel activeJobTableModel = (DefaultTableModel) table.getModel();
long jobId = Long.valueOf(activeJobTableModel.getValueAt(modelRow, HpcJobActivityEditor.ID_COLUMN).toString()).longValue();
int answer = JOptionPane.showConfirmDialog(parentComp, "Would you like to cancel this HPC job id: " + jobId + "?", "Cancel HPC job", JOptionPane.YES_NO_OPTION);
if (answer == JOptionPane.NO_OPTION)
return;
TetradDesktop desktop = (TetradDesktop) DesktopController.getInstance();
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
HpcJobInfo hpcJobInfo = hpcJobManager.findHpcJobInfoById(Long.valueOf(activeJobTableModel.getValueAt(modelRow, HpcJobActivityEditor.ID_COLUMN).toString()).longValue());
if (hpcJobInfo != null) {
try {
if (hpcJobInfo.getPid() != null) {
// Update table
activeJobTableModel.setValueAt("Kill Request", modelRow, 1);
table.updateUI();
hpcJobInfo = hpcJobManager.requestHpcJobKilled(hpcJobInfo);
// Update hpcJobInfo instance
hpcJobManager.updateHpcJobInfo(hpcJobInfo);
// Update hpcJobLog instance
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
if (hpcJobLog != null) {
hpcJobLog.setLastUpdatedTime(new Date(System.currentTimeMillis()));
hpcJobManager.updateHpcJobLog(hpcJobLog);
// Update hpcJobLogDetail instance
String log = "Requested job id " + hpcJobLog.getId() + " killed";
hpcJobManager.logHpcJobLogDetail(hpcJobLog, 2, log);
}
} else {
// Update table
activeJobTableModel.removeRow(modelRow);
table.updateUI();
hpcJobManager.removePendingHpcJob(hpcJobInfo);
// Killed
hpcJobInfo.setStatus(4);
// Update hpcJobInfo instance
hpcJobManager.updateHpcJobInfo(hpcJobInfo);
// Update hpcJobLog instance
HpcJobLog hpcJobLog = hpcJobManager.getHpcJobLog(hpcJobInfo);
if (hpcJobLog != null) {
hpcJobLog.setCanceledTime(new Date(System.currentTimeMillis()));
hpcJobLog.setLastUpdatedTime(new Date(System.currentTimeMillis()));
hpcJobManager.updateHpcJobLog(hpcJobLog);
// Update hpcJobLogDetail instance
String log = "Killed job id " + hpcJobLog.getId();
hpcJobManager.logHpcJobLogDetail(hpcJobLog, 4, log);
}
}
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
use of edu.cmu.tetradapp.app.hpc.manager.HpcJobManager in project tetrad by cmu-phil.
the class LoadHpcGraphJsonAction method buildHpcJsonChooserComponent.
private JComponent buildHpcJsonChooserComponent(final TetradDesktop desktop) {
final HpcAccountManager hpcAccountManager = desktop.getHpcAccountManager();
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
// Get ComputingAccount from DB
final DefaultListModel<HpcAccount> listModel = new DefaultListModel<HpcAccount>();
for (HpcAccount account : hpcAccountManager.getHpcAccounts()) {
listModel.addElement(account);
}
// JSplitPane
final JSplitPane splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
// Left pane -> JList (parent pane)
JPanel leftPanel = new JPanel(new BorderLayout());
// Right pane -> ComputingAccountResultList
final JPanel jsonResultListPanel = new JPanel(new BorderLayout());
int minWidth = 800;
int minHeight = 600;
int screenWidth = Toolkit.getDefaultToolkit().getScreenSize().width;
int screenHeight = Toolkit.getDefaultToolkit().getScreenSize().height;
int frameWidth = screenWidth * 3 / 4;
int frameHeight = screenHeight * 3 / 4;
final int paneWidth = minWidth > frameWidth ? minWidth : frameWidth;
final int paneHeight = minHeight > frameHeight ? minHeight : frameHeight;
// JTable
final Vector<String> columnNames = new Vector<>();
columnNames.addElement("Name");
columnNames.addElement("Created");
columnNames.addElement("Last Modified");
columnNames.addElement("Size");
Vector<Vector<String>> rowData = new Vector<>();
final DefaultTableModel tableModel = new LoadHpcGraphJsonTableModel(rowData, columnNames);
final JTable jsonResultTable = new JTable(tableModel);
jsonResultTable.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
// Resize table's column width
jsonResultTable.getColumnModel().getColumn(0).setPreferredWidth(paneWidth * 2 / 5);
jsonResultTable.getColumnModel().getColumn(1).setPreferredWidth(paneWidth * 2 / 15);
jsonResultTable.getColumnModel().getColumn(2).setPreferredWidth(paneWidth * 2 / 15);
jsonResultTable.getColumnModel().getColumn(3).setPreferredWidth(paneWidth * 2 / 15);
ListSelectionModel selectionModel = jsonResultTable.getSelectionModel();
selectionModel.addListSelectionListener(new ListSelectionListener() {
@Override
public void valueChanged(ListSelectionEvent e) {
int row = jsonResultTable.getSelectedRow();
if (row >= 0) {
DefaultTableModel model = (DefaultTableModel) jsonResultTable.getModel();
jsonFileName = (String) model.getValueAt(row, 0);
}
}
});
final JScrollPane scrollTablePane = new JScrollPane(jsonResultTable);
jsonResultListPanel.add(scrollTablePane, BorderLayout.CENTER);
splitPane.setLeftComponent(leftPanel);
splitPane.setRightComponent(jsonResultListPanel);
// Center Panel
final JList<HpcAccount> accountList = new JList<>(listModel);
accountList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
accountList.setLayoutOrientation(JList.VERTICAL);
accountList.setSelectedIndex(-1);
accountList.addListSelectionListener(new ListSelectionListener() {
@Override
public void valueChanged(ListSelectionEvent e) {
if (e.getValueIsAdjusting())
return;
int selectedIndex = ((JList<?>) e.getSource()).getSelectedIndex();
// Show or remove the json list
if (selectedIndex > -1) {
jsonFileName = null;
hpcAccount = listModel.get(selectedIndex);
TableColumnModel columnModel = jsonResultTable.getColumnModel();
List<Integer> columnWidthList = new ArrayList<>();
for (int i = 0; i < columnModel.getColumnCount(); i++) {
int width = columnModel.getColumn(i).getPreferredWidth();
columnWidthList.add(width);
}
jsonResultTable.clearSelection();
try {
HpcAccountService hpcAccountService = hpcJobManager.getHpcAccountService(hpcAccount);
ResultService resultService = hpcAccountService.getResultService();
Set<ResultFile> results = resultService.listAlgorithmResultFiles(HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
Vector<Vector<String>> jsonFiles = new Vector<>();
for (ResultFile resultFile : results) {
if (resultFile.getName().endsWith(".json")) {
Vector<String> rowData = new Vector<>();
rowData.addElement(resultFile.getName());
rowData.addElement(FilePrint.fileTimestamp(resultFile.getCreationTime().getTime()));
rowData.addElement(FilePrint.fileTimestamp(resultFile.getLastModifiedTime().getTime()));
rowData.addElement(FilePrint.humanReadableSize(resultFile.getFileSize(), false));
jsonFiles.add(rowData);
}
}
tableModel.setDataVector(jsonFiles, columnNames);
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
// Resize table's column width
for (int i = 0; i < columnModel.getColumnCount(); i++) {
jsonResultTable.getColumnModel().getColumn(i).setPreferredWidth(columnWidthList.get(i).intValue());
}
}
}
});
// Left Panel
JScrollPane accountListScroller = new JScrollPane(accountList);
leftPanel.add(accountListScroller, BorderLayout.CENTER);
splitPane.setDividerLocation(paneWidth / 5);
accountListScroller.setPreferredSize(new Dimension(paneWidth / 5, paneHeight));
jsonResultListPanel.setPreferredSize(new Dimension(paneWidth * 4 / 5, paneHeight));
return splitPane;
}
use of edu.cmu.tetradapp.app.hpc.manager.HpcJobManager in project tetrad by cmu-phil.
the class LoadHpcGraphJsonAction method actionPerformed.
@Override
public void actionPerformed(ActionEvent e) {
TetradDesktop desktop = (TetradDesktop) DesktopController.getInstance();
final HpcAccountManager hpcAccountManager = desktop.getHpcAccountManager();
final HpcJobManager hpcJobManager = desktop.getHpcJobManager();
JComponent comp = buildHpcJsonChooserComponent(desktop);
int option = JOptionPane.showConfirmDialog(JOptionUtils.centeringComp(), comp, "High-Performance Computing Account Json Results Chooser", JOptionPane.OK_CANCEL_OPTION, JOptionPane.PLAIN_MESSAGE);
if (option == JOptionPane.OK_OPTION && jsonFileName != null && hpcAccount != null) {
try {
HpcAccountService hpcAccountService = hpcJobManager.getHpcAccountService(hpcAccount);
ResultService resultService = hpcAccountService.getResultService();
String json = resultService.downloadAlgorithmResultFile(jsonFileName, HpcAccountUtils.getJsonWebToken(hpcAccountManager, hpcAccount));
Graph graph = JsonUtils.parseJSONObjectToTetradGraph(json);
GraphUtils.circleLayout(graph, 300, 300, 150);
graphEditable.setGraph(graph);
graphEditable.setName(jsonFileName);
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
} else {
LOGGER.debug("Option: OK " + (option == JOptionPane.OK_OPTION));
LOGGER.debug("Option: jsonFileName " + (jsonFileName != null));
LOGGER.debug("Option: computingAccount " + (hpcAccount != null));
}
}
Aggregations