Examples with JobStatus org.apache.samza.rest.model.JobStatus used on opensource projects

Example 1 with JobStatus

use of org.apache.samza.rest.model.JobStatus in project samza by apache.

the class SimpleYarnJobProxy method stop.

@Override
public void stop(JobInstance jobInstance) throws Exception {
    JobStatus currentStatus = getJobSamzaStatus(jobInstance);
    if (!currentStatus.hasBeenStarted()) {
        log.info("Job {} will not be stopped because it is currently {}.", jobInstance, currentStatus.toString());
        return;
    }
    String scriptPath = getScriptPath(jobInstance, STOP_SCRIPT_NAME);
    int resultCode = scriptRunner.runScript(scriptPath, YarnCliJobStatusProvider.getQualifiedJobName(jobInstance));
    if (resultCode != 0) {
        throw new SamzaException("Failed to stop job. Result code: " + resultCode);
    }
}

Example 2 with JobStatus

use of org.apache.samza.rest.model.JobStatus in project samza by apache.

the class JobsResource method updateJobStatus.

/**
   *
   * @param jobName the name of the job as configured in {@link org.apache.samza.config.JobConfig#JOB_NAME}.
   * @param jobId   the id of the job as configured in {@link org.apache.samza.config.JobConfig#JOB_ID}.
   * @param status   the {@link JobStatus} to which the job will transition.
   * @return        a {@link javax.ws.rs.core.Response.Status#ACCEPTED} {@link javax.ws.rs.core.Response}
   *                containing a {@link Job} for the Samza job if it is
   *                installed on this host. {@link javax.ws.rs.core.Response.Status#NOT_FOUND}
   *                {@link javax.ws.rs.core.Response.Status#BAD_REQUEST} and
   *                {@link javax.ws.rs.core.Response.Status#INTERNAL_SERVER_ERROR} can occur for corresponding errors.
   */
@PUT
@Path("/{jobName}/{jobId}")
@Produces(MediaType.APPLICATION_JSON)
public Response updateJobStatus(@PathParam("jobName") final String jobName, @PathParam("jobId") final String jobId, @QueryParam("status") String status) {
    JobInstance jobInstance = new JobInstance(jobName, jobId);
    try {
        if (!jobProxy.jobExists(jobInstance)) {
            return Response.status(Response.Status.NOT_FOUND).entity(Collections.singletonMap("message", String.format("Job %s instance %s is not installed on this host.", jobName, jobId))).build();
        }
        if (status == null) {
            throw new IllegalArgumentException("Unrecognized status parameter: " + status);
        }
        JobStatus samzaStatus = JobStatus.valueOf(status.toUpperCase());
        switch(samzaStatus) {
            case STARTED:
                log.info("Starting {}", jobInstance);
                jobProxy.start(jobInstance);
                Job infoStarted = jobProxy.getJobStatus(jobInstance);
                return Response.accepted(infoStarted).build();
            case STOPPED:
                log.info("Stopping {}", jobInstance);
                jobProxy.stop(jobInstance);
                Job infoStopped = jobProxy.getJobStatus(jobInstance);
                return Response.accepted(infoStopped).build();
            default:
                throw new IllegalArgumentException("Unsupported status: " + status);
        }
    } catch (IllegalArgumentException e) {
        log.info(String.format("Illegal arguments updateJobStatus. JobName:%s JobId:%s Status=%s", jobName, jobId, status), e);
        return Responses.badRequestResponse(e.getMessage());
    } catch (Exception e) {
        log.error("Error in updateJobStatus.", e);
        return Responses.errorResponse(String.format("Error type: %s message: %s", e.toString(), e.getMessage()));
    }
}

Example 3 with JobStatus

use of org.apache.samza.rest.model.JobStatus in project samza by apache.

the class SimpleYarnJobProxy method start.

@Override
public void start(JobInstance jobInstance) throws Exception {
    JobStatus currentStatus = getJobSamzaStatus(jobInstance);
    if (currentStatus.hasBeenStarted()) {
        log.info("Job {} will not be started because it is currently {}.", jobInstance, currentStatus.toString());
        return;
    }
    String scriptPath = getScriptPath(jobInstance, START_SCRIPT_NAME);
    int resultCode = scriptRunner.runScript(scriptPath, CONFIG_FACTORY_PARAM, generateConfigPathParameter(jobInstance));
    if (resultCode != 0) {
        throw new SamzaException("Failed to start job. Result code: " + resultCode);
    }
}

Example 4 with JobStatus

use of org.apache.samza.rest.model.JobStatus in project samza by apache.

the class YarnCliJobStatusProvider method getJobStatuses.

@Override
public void getJobStatuses(Collection<Job> jobs) throws IOException, InterruptedException {
    if (jobs == null || jobs.isEmpty()) {
        return;
    }
    // If the scripts are in the jobs, they will be in all job installations, so just pick one and get the script path.
    Job anyJob = jobs.iterator().next();
    String scriptPath = scriptPathProvider.getScriptPath(new JobInstance(anyJob.getJobName(), anyJob.getJobId()), "run-class.sh");
    // We will identify jobs returned by the YARN application states by their qualified names, so build a map
    // to translate back from that name to the JobInfo we wish to populate. This avoids parsing/delimiter issues.
    final Map<String, Job> qualifiedJobToInfo = new HashMap<>();
    for (Job job : jobs) {
        qualifiedJobToInfo.put(getQualifiedJobName(new JobInstance(job.getJobName(), job.getJobId())), job);
    }
    // Run "application -list" command and get the YARN state for each application
    ScriptRunner runner = new ScriptRunner();
    int resultCode = runner.runScript(scriptPath, new ScriptOutputHandler() {

        @Override
        public void processScriptOutput(InputStream output) throws IOException {
            InputStreamReader isr = new InputStreamReader(output);
            BufferedReader br = new BufferedReader(isr);
            String line;
            String APPLICATION_PREFIX = "application_";
            log.debug("YARN status:");
            while ((line = br.readLine()) != null) {
                log.debug(line);
                if (line.startsWith(APPLICATION_PREFIX)) {
                    String[] columns = line.split("\\s+");
                    String qualifiedName = columns[1];
                    String yarnState = columns[5];
                    JobStatus samzaStatus = yarnStateToSamzaStatus(YarnApplicationState.valueOf(yarnState.toUpperCase()));
                    Job job = qualifiedJobToInfo.get(qualifiedName);
                    // application attempts in that status. Only update the job status if it's not STOPPED.
                    if (job != null && (job.getStatusDetail() == null || samzaStatus != JobStatus.STOPPED)) {
                        job.setStatusDetail(yarnState);
                        job.setStatus(samzaStatus);
                    }
                }
            }
        }
    }, "org.apache.hadoop.yarn.client.cli.ApplicationCLI", "application", "-list", "-appStates", "ALL");
    if (resultCode != 0) {
        throw new SamzaException("Failed to get job status. Result code: " + resultCode);
    }
}

Example 5 with JobStatus

use of org.apache.samza.rest.model.JobStatus in project samza by apache.

the class LocalStoreMonitor method monitor.

/**
   * This monitor method is invoked periodically to delete the stale state stores
   * of dead jobs/tasks.
   * @throws Exception if there was any problem in running the monitor.
   */
@Override
public void monitor() throws Exception {
    File localStoreDir = new File(config.getLocalStoreBaseDir());
    Preconditions.checkState(localStoreDir.isDirectory(), String.format("LocalStoreDir: %s is not a directory", localStoreDir.getAbsolutePath()));
    String localHostName = InetAddress.getLocalHost().getHostName();
    for (JobInstance jobInstance : getHostAffinityEnabledJobs(localStoreDir)) {
        File jobDir = new File(localStoreDir, String.format("%s-%s", jobInstance.getJobName(), jobInstance.getJobId()));
        try {
            JobStatus jobStatus = jobsClient.getJobStatus(jobInstance);
            for (Task task : jobsClient.getTasks(jobInstance)) {
                for (String storeName : jobDir.list(DirectoryFileFilter.DIRECTORY)) {
                    LOG.info("Job: {} has the running status: {} with preferred host: {}.", jobInstance, jobStatus, task.getPreferredHost());
                    /**
             *  A task store is active if all of the following conditions are true:
             *  a) If the store is amongst the active stores of the task.
             *  b) If the job has been started.
             *  c) If the preferred host of the task is the localhost on which the monitor is run.
             */
                    if (jobStatus.hasBeenStarted() && task.getStoreNames().contains(storeName) && task.getPreferredHost().equals(localHostName)) {
                        LOG.info(String.format("Store %s is actively used by the task: %s.", storeName, task.getTaskName()));
                    } else {
                        LOG.info(String.format("Store %s not used by the task: %s.", storeName, task.getTaskName()));
                        markSweepTaskStore(TaskStorageManager.getStorePartitionDir(jobDir, storeName, new TaskName(task.getTaskName())));
                    }
                }
            }
        } catch (Exception ex) {
            if (!config.getIgnoreFailures()) {
                throw ex;
            }
            LOG.warn("Config: {} turned on, failures will be ignored. Local store cleanup for job: {} resulted in exception: {}.", new Object[] { LocalStoreMonitorConfig.CONFIG_IGNORE_FAILURES, jobInstance, ex });
        }
    }
}