Examples with JobRunner org.apache.samza.job.JobRunner used on opensource projects

Example 1 with JobRunner

use of org.apache.samza.job.JobRunner in project samza by apache.

the class RemoteApplicationRunner method status.

@Override
public ApplicationStatus status(StreamApplication app) {
    try {
        boolean hasNewJobs = false;
        boolean hasRunningJobs = false;
        ApplicationStatus unsuccessfulFinishStatus = null;
        ExecutionPlan plan = getExecutionPlan(app);
        for (JobConfig jobConfig : plan.getJobConfigs()) {
            JobRunner runner = new JobRunner(jobConfig);
            ApplicationStatus status = runner.status();
            log.debug("Status is {} for job {}", new Object[] { status, jobConfig.getName() });
            switch(status.getStatusCode()) {
                case New:
                    hasNewJobs = true;
                    break;
                case Running:
                    hasRunningJobs = true;
                    break;
                case UnsuccessfulFinish:
                    unsuccessfulFinishStatus = status;
                    break;
                case SuccessfulFinish:
                    break;
                default:
            }
        }
        if (hasNewJobs) {
            // There are jobs not started, report as New
            return ApplicationStatus.New;
        } else if (hasRunningJobs) {
            // All jobs are started, some are running
            return ApplicationStatus.Running;
        } else if (unsuccessfulFinishStatus != null) {
            // All jobs are finished, some are not successful
            return unsuccessfulFinishStatus;
        } else {
            // All jobs are finished successfully
            return ApplicationStatus.SuccessfulFinish;
        }
    } catch (Throwable t) {
        throw new SamzaException("Failed to get status for application", t);
    }
}

Example 2 with JobRunner

use of org.apache.samza.job.JobRunner in project samza by apache.

the class ConfigManager method handleYarnContainerChange.

/**
   * This method handles setConfig messages that want to change the number of containers of a job
   *
   * @param containerCountAsString the new number of containers in a String format
   */
private void handleYarnContainerChange(String containerCountAsString) throws IOException, YarnException {
    String applicationId = yarnUtil.getRunningAppId(jobName, jobID);
    int containerCount = Integer.valueOf(containerCountAsString);
    //checking the input is valid
    int currentNumTask = getCurrentNumTasks();
    int currentNumContainers = getCurrentNumContainers();
    if (containerCount == currentNumContainers) {
        log.error("The new number of containers is equal to the current number of containers, skipping this message");
        return;
    }
    if (containerCount <= 0) {
        log.error("The number of containers cannot be zero or less, skipping this message");
        return;
    }
    if (containerCount > currentNumTask) {
        log.error("The number of containers cannot be more than the number of task, skipping this message");
        return;
    }
    //killing the current job
    log.info("Killing the current job");
    yarnUtil.killApplication(applicationId);
    //reset the global variables
    coordinatorServerURL = null;
    try {
        //waiting for the job to be killed
        String state = yarnUtil.getApplicationState(applicationId);
        Thread.sleep(1000);
        int countSleep = 1;
        while (!state.equals("KILLED")) {
            state = yarnUtil.getApplicationState(applicationId);
            log.info("Job kill signal sent, but job not killed yet for " + applicationId + ". Sleeping for another 1000ms");
            Thread.sleep(1000);
            countSleep++;
            if (countSleep > 10) {
                throw new IllegalStateException("Job has not been killed after 10 attempts.");
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    log.info("Killed the current job successfully");
    //start the job again
    log.info("Staring the job again");
    skipUnreadMessages();
    JobRunner jobRunner = new JobRunner(config);
    jobRunner.run(false);
}

Example 3 with JobRunner

use of org.apache.samza.job.JobRunner in project samza by apache.

the class RemoteApplicationRunner method run.

/**
   * Run the {@link StreamApplication} on the remote cluster
   * @param app a StreamApplication
   */
@Override
public void run(StreamApplication app) {
    try {
        // 1. initialize and plan
        ExecutionPlan plan = getExecutionPlan(app);
        writePlanJsonFile(plan.getPlanAsJson());
        // 2. create the necessary streams
        getStreamManager().createStreams(plan.getIntermediateStreams());
        // 3. submit jobs for remote execution
        plan.getJobConfigs().forEach(jobConfig -> {
            log.info("Starting job {} with config {}", jobConfig.getName(), jobConfig);
            JobRunner runner = new JobRunner(jobConfig);
            runner.run(true);
        });
    } catch (Throwable t) {
        throw new SamzaException("Failed to run application", t);
    }
}

Example 4 with JobRunner

use of org.apache.samza.job.JobRunner in project samza by apache.

the class RemoteApplicationRunner method kill.

@Override
public void kill(StreamApplication app) {
    try {
        ExecutionPlan plan = getExecutionPlan(app);
        plan.getJobConfigs().forEach(jobConfig -> {
            log.info("Killing job {}", jobConfig.getName());
            JobRunner runner = new JobRunner(jobConfig);
            runner.kill();
        });
    } catch (Throwable t) {
        throw new SamzaException("Failed to kill application", t);
    }
}