Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 46 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class SimpleYarnJobProxy method start.

@Override
public void start(JobInstance jobInstance) throws Exception {
    JobStatus currentStatus = getJobSamzaStatus(jobInstance);
    if (currentStatus.hasBeenStarted()) {
        log.info("Job {} will not be started because it is currently {}.", jobInstance, currentStatus.toString());
        return;
    }
    String scriptPath = getScriptPath(jobInstance, START_SCRIPT_NAME);
    int resultCode = scriptRunner.runScript(scriptPath, CONFIG_FACTORY_PARAM, generateConfigPathParameter(jobInstance));
    if (resultCode != 0) {
        throw new SamzaException("Failed to start job. Result code: " + resultCode);
    }
}

Example 47 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class YarnCliJobStatusProvider method getJobStatuses.

@Override
public void getJobStatuses(Collection<Job> jobs) throws IOException, InterruptedException {
    if (jobs == null || jobs.isEmpty()) {
        return;
    }
    // If the scripts are in the jobs, they will be in all job installations, so just pick one and get the script path.
    Job anyJob = jobs.iterator().next();
    String scriptPath = scriptPathProvider.getScriptPath(new JobInstance(anyJob.getJobName(), anyJob.getJobId()), "run-class.sh");
    // We will identify jobs returned by the YARN application states by their qualified names, so build a map
    // to translate back from that name to the JobInfo we wish to populate. This avoids parsing/delimiter issues.
    final Map<String, Job> qualifiedJobToInfo = new HashMap<>();
    for (Job job : jobs) {
        qualifiedJobToInfo.put(getQualifiedJobName(new JobInstance(job.getJobName(), job.getJobId())), job);
    }
    // Run "application -list" command and get the YARN state for each application
    ScriptRunner runner = new ScriptRunner();
    int resultCode = runner.runScript(scriptPath, new ScriptOutputHandler() {

        @Override
        public void processScriptOutput(InputStream output) throws IOException {
            InputStreamReader isr = new InputStreamReader(output);
            BufferedReader br = new BufferedReader(isr);
            String line;
            String APPLICATION_PREFIX = "application_";
            log.debug("YARN status:");
            while ((line = br.readLine()) != null) {
                log.debug(line);
                if (line.startsWith(APPLICATION_PREFIX)) {
                    String[] columns = line.split("\\s+");
                    String qualifiedName = columns[1];
                    String yarnState = columns[5];
                    JobStatus samzaStatus = yarnStateToSamzaStatus(YarnApplicationState.valueOf(yarnState.toUpperCase()));
                    Job job = qualifiedJobToInfo.get(qualifiedName);
                    // application attempts in that status. Only update the job status if it's not STOPPED.
                    if (job != null && (job.getStatusDetail() == null || samzaStatus != JobStatus.STOPPED)) {
                        job.setStatusDetail(yarnState);
                        job.setStatus(samzaStatus);
                    }
                }
            }
        }
    }, "org.apache.hadoop.yarn.client.cli.ApplicationCLI", "application", "-list", "-appStates", "ALL");
    if (resultCode != 0) {
        throw new SamzaException("Failed to get job status. Result code: " + resultCode);
    }
}

Example 48 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class SamzaMonitorService method start.

public void start() {
    try {
        Map<String, MonitorConfig> monitorConfigs = getMonitorConfigs(config);
        for (Map.Entry<String, MonitorConfig> entry : monitorConfigs.entrySet()) {
            String monitorName = entry.getKey();
            MonitorConfig monitorConfig = entry.getValue();
            if (!Strings.isNullOrEmpty(monitorConfig.getMonitorFactoryClass())) {
                int schedulingIntervalInMs = monitorConfig.getSchedulingIntervalInMs();
                LOGGER.info("Scheduling monitor {} to run every {} ms", monitorName, schedulingIntervalInMs);
                // MetricsRegistry has been added in the Monitor interface, since it's required in the eventual future to record metrics.
                // We have plans to record metrics, hence adding this as a placeholder. We just aren't doing it yet.
                scheduler.schedule(getRunnable(instantiateMonitor(monitorName, monitorConfig, metricsRegistry)), schedulingIntervalInMs);
            } else {
                // When MonitorFactoryClass is not defined in the config, ignore the monitor config
                LOGGER.warn("Not scheduling the monitor: {} to run, since monitor factory class is not set in config.", monitorName);
            }
        }
    } catch (InstantiationException e) {
        LOGGER.error("Exception when instantiating the monitor : ", e);
        throw new SamzaException(e);
    }
}

Example 49 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StreamAppender method getConfig.

/**
   * get the config for the AM or containers based on the containers' names.
   *
   * @return Config the config of this container
   */
protected Config getConfig() {
    Config config = null;
    try {
        if (isApplicationMaster) {
            config = JobModelManager.currentJobModelManager().jobModel().getConfig();
        } else {
            String url = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
            config = SamzaObjectMapper.getObjectMapper().readValue(Util.read(new URL(url), 30000), JobModel.class).getConfig();
        }
    } catch (IOException e) {
        throw new SamzaException("can not read the config", e);
    }
    return config;
}

Example 50 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class GroupByContainerCount method getPreviousContainers.

/**
   * Reads the task-container mapping from the provided {@link TaskAssignmentManager} and returns a
   * list of TaskGroups, ordered ascending by containerId.
   *
   * @param taskAssignmentManager the {@link TaskAssignmentManager} that will be used to retrieve the previous mapping.
   * @param taskCount             the number of tasks, for validation against the persisted tasks.
   * @return                      a list of TaskGroups, ordered ascending by containerId or {@code null}
   *                              if the previous mapping doesn't exist or isn't usable.
   */
private List<TaskGroup> getPreviousContainers(TaskAssignmentManager taskAssignmentManager, int taskCount) {
    Map<String, String> taskToContainerId = taskAssignmentManager.readTaskAssignment();
    taskToContainerId.values().forEach(id -> {
        try {
            int intId = Integer.parseInt(id);
        } catch (NumberFormatException nfe) {
            throw new SamzaException("GroupByContainerCount cannot handle non-integer processorIds!", nfe);
        }
    });
    if (taskToContainerId.isEmpty()) {
        log.info("No task assignment map was saved.");
        return null;
    } else if (taskCount != taskToContainerId.size()) {
        log.warn("Current task count {} does not match saved task count {}. Stateful jobs may observe misalignment of keys!", taskCount, taskToContainerId.size());
        // If the tasks changed, then the partition-task grouping is also likely changed and we can't handle that
        // without a much more complicated mapping. Further, the partition count may have changed, which means
        // input message keys are likely reshuffled w.r.t. partitions, so the local state may not contain necessary
        // data associated with the incoming keys. Warn the user and default to grouper
        // In this scenario the tasks may have been reduced, so we need to delete all the existing messages
        taskAssignmentManager.deleteTaskContainerMappings(taskToContainerId.keySet());
        return null;
    }
    List<TaskGroup> containers;
    try {
        containers = getOrderedContainers(taskToContainerId);
    } catch (Exception e) {
        log.error("Exception while parsing task mapping", e);
        return null;
    }
    return containers;
}