use of org.apache.samza.rest.script.ScriptOutputHandler in project samza by apache.
the class YarnCliJobStatusProvider method getJobStatuses.
@Override
public void getJobStatuses(Collection<Job> jobs) throws IOException, InterruptedException {
if (jobs == null || jobs.isEmpty()) {
return;
}
// If the scripts are in the jobs, they will be in all job installations, so just pick one and get the script path.
Job anyJob = jobs.iterator().next();
String scriptPath = scriptPathProvider.getScriptPath(new JobInstance(anyJob.getJobName(), anyJob.getJobId()), "run-class.sh");
// We will identify jobs returned by the YARN application states by their qualified names, so build a map
// to translate back from that name to the JobInfo we wish to populate. This avoids parsing/delimiter issues.
final Map<String, Job> qualifiedJobToInfo = new HashMap<>();
for (Job job : jobs) {
qualifiedJobToInfo.put(getQualifiedJobName(new JobInstance(job.getJobName(), job.getJobId())), job);
}
// Run "application -list" command and get the YARN state for each application
ScriptRunner runner = new ScriptRunner();
int resultCode = runner.runScript(scriptPath, new ScriptOutputHandler() {
@Override
public void processScriptOutput(InputStream output) throws IOException {
InputStreamReader isr = new InputStreamReader(output);
BufferedReader br = new BufferedReader(isr);
String line;
String APPLICATION_PREFIX = "application_";
log.debug("YARN status:");
while ((line = br.readLine()) != null) {
log.debug(line);
if (line.startsWith(APPLICATION_PREFIX)) {
String[] columns = line.split("\\s+");
String qualifiedName = columns[1];
String yarnState = columns[5];
JobStatus samzaStatus = yarnStateToSamzaStatus(YarnApplicationState.valueOf(yarnState.toUpperCase()));
Job job = qualifiedJobToInfo.get(qualifiedName);
// application attempts in that status. Only update the job status if it's not STOPPED.
if (job != null && (job.getStatusDetail() == null || samzaStatus != JobStatus.STOPPED)) {
job.setStatusDetail(yarnState);
job.setStatus(samzaStatus);
}
}
}
}
}, "org.apache.hadoop.yarn.client.cli.ApplicationCLI", "application", "-list", "-appStates", "ALL");
if (resultCode != 0) {
throw new SamzaException("Failed to get job status. Result code: " + resultCode);
}
}
Aggregations