Examples with ScriptOutputHandler org.apache.samza.rest.script.ScriptOutputHandler used on opensource projects

Example 1 with ScriptOutputHandler

use of org.apache.samza.rest.script.ScriptOutputHandler in project samza by apache.

the class YarnCliJobStatusProvider method getJobStatuses.

@Override
public void getJobStatuses(Collection<Job> jobs) throws IOException, InterruptedException {
    if (jobs == null || jobs.isEmpty()) {
        return;
    }
    // If the scripts are in the jobs, they will be in all job installations, so just pick one and get the script path.
    Job anyJob = jobs.iterator().next();
    String scriptPath = scriptPathProvider.getScriptPath(new JobInstance(anyJob.getJobName(), anyJob.getJobId()), "run-class.sh");
    // We will identify jobs returned by the YARN application states by their qualified names, so build a map
    // to translate back from that name to the JobInfo we wish to populate. This avoids parsing/delimiter issues.
    final Map<String, Job> qualifiedJobToInfo = new HashMap<>();
    for (Job job : jobs) {
        qualifiedJobToInfo.put(getQualifiedJobName(new JobInstance(job.getJobName(), job.getJobId())), job);
    }
    // Run "application -list" command and get the YARN state for each application
    ScriptRunner runner = new ScriptRunner();
    int resultCode = runner.runScript(scriptPath, new ScriptOutputHandler() {

        @Override
        public void processScriptOutput(InputStream output) throws IOException {
            InputStreamReader isr = new InputStreamReader(output);
            BufferedReader br = new BufferedReader(isr);
            String line;
            String APPLICATION_PREFIX = "application_";
            log.debug("YARN status:");
            while ((line = br.readLine()) != null) {
                log.debug(line);
                if (line.startsWith(APPLICATION_PREFIX)) {
                    String[] columns = line.split("\\s+");
                    String qualifiedName = columns[1];
                    String yarnState = columns[5];
                    JobStatus samzaStatus = yarnStateToSamzaStatus(YarnApplicationState.valueOf(yarnState.toUpperCase()));
                    Job job = qualifiedJobToInfo.get(qualifiedName);
                    // application attempts in that status. Only update the job status if it's not STOPPED.
                    if (job != null && (job.getStatusDetail() == null || samzaStatus != JobStatus.STOPPED)) {
                        job.setStatusDetail(yarnState);
                        job.setStatus(samzaStatus);
                    }
                }
            }
        }
    }, "org.apache.hadoop.yarn.client.cli.ApplicationCLI", "application", "-list", "-appStates", "ALL");
    if (resultCode != 0) {
        throw new SamzaException("Failed to get job status. Result code: " + resultCode);
    }
}