Examples with JobInstance org.apache.samza.rest.proxy.job.JobInstance used on opensource projects

Example 6 with JobInstance

use of org.apache.samza.rest.proxy.job.JobInstance in project samza by apache.

the class LocalStoreMonitor method monitor.

/**
   * This monitor method is invoked periodically to delete the stale state stores
   * of dead jobs/tasks.
   * @throws Exception if there was any problem in running the monitor.
   */
@Override
public void monitor() throws Exception {
    File localStoreDir = new File(config.getLocalStoreBaseDir());
    Preconditions.checkState(localStoreDir.isDirectory(), String.format("LocalStoreDir: %s is not a directory", localStoreDir.getAbsolutePath()));
    String localHostName = InetAddress.getLocalHost().getHostName();
    for (JobInstance jobInstance : getHostAffinityEnabledJobs(localStoreDir)) {
        File jobDir = new File(localStoreDir, String.format("%s-%s", jobInstance.getJobName(), jobInstance.getJobId()));
        try {
            JobStatus jobStatus = jobsClient.getJobStatus(jobInstance);
            for (Task task : jobsClient.getTasks(jobInstance)) {
                for (String storeName : jobDir.list(DirectoryFileFilter.DIRECTORY)) {
                    LOG.info("Job: {} has the running status: {} with preferred host: {}.", jobInstance, jobStatus, task.getPreferredHost());
                    /**
             *  A task store is active if all of the following conditions are true:
             *  a) If the store is amongst the active stores of the task.
             *  b) If the job has been started.
             *  c) If the preferred host of the task is the localhost on which the monitor is run.
             */
                    if (jobStatus.hasBeenStarted() && task.getStoreNames().contains(storeName) && task.getPreferredHost().equals(localHostName)) {
                        LOG.info(String.format("Store %s is actively used by the task: %s.", storeName, task.getTaskName()));
                    } else {
                        LOG.info(String.format("Store %s not used by the task: %s.", storeName, task.getTaskName()));
                        markSweepTaskStore(TaskStorageManager.getStorePartitionDir(jobDir, storeName, new TaskName(task.getTaskName())));
                    }
                }
            }
        } catch (Exception ex) {
            if (!config.getIgnoreFailures()) {
                throw ex;
            }
            LOG.warn("Config: {} turned on, failures will be ignored. Local store cleanup for job: {} resulted in exception: {}.", new Object[] { LocalStoreMonitorConfig.CONFIG_IGNORE_FAILURES, jobInstance, ex });
        }
    }
}