Search in sources :

Example 6 with JobInstance

use of org.apache.samza.rest.proxy.job.JobInstance in project samza by apache.

the class LocalStoreMonitor method monitor.

/**
   * This monitor method is invoked periodically to delete the stale state stores
   * of dead jobs/tasks.
   * @throws Exception if there was any problem in running the monitor.
   */
@Override
public void monitor() throws Exception {
    File localStoreDir = new File(config.getLocalStoreBaseDir());
    Preconditions.checkState(localStoreDir.isDirectory(), String.format("LocalStoreDir: %s is not a directory", localStoreDir.getAbsolutePath()));
    String localHostName = InetAddress.getLocalHost().getHostName();
    for (JobInstance jobInstance : getHostAffinityEnabledJobs(localStoreDir)) {
        File jobDir = new File(localStoreDir, String.format("%s-%s", jobInstance.getJobName(), jobInstance.getJobId()));
        try {
            JobStatus jobStatus = jobsClient.getJobStatus(jobInstance);
            for (Task task : jobsClient.getTasks(jobInstance)) {
                for (String storeName : jobDir.list(DirectoryFileFilter.DIRECTORY)) {
                    LOG.info("Job: {} has the running status: {} with preferred host: {}.", jobInstance, jobStatus, task.getPreferredHost());
                    /**
             *  A task store is active if all of the following conditions are true:
             *  a) If the store is amongst the active stores of the task.
             *  b) If the job has been started.
             *  c) If the preferred host of the task is the localhost on which the monitor is run.
             */
                    if (jobStatus.hasBeenStarted() && task.getStoreNames().contains(storeName) && task.getPreferredHost().equals(localHostName)) {
                        LOG.info(String.format("Store %s is actively used by the task: %s.", storeName, task.getTaskName()));
                    } else {
                        LOG.info(String.format("Store %s not used by the task: %s.", storeName, task.getTaskName()));
                        markSweepTaskStore(TaskStorageManager.getStorePartitionDir(jobDir, storeName, new TaskName(task.getTaskName())));
                    }
                }
            }
        } catch (Exception ex) {
            if (!config.getIgnoreFailures()) {
                throw ex;
            }
            LOG.warn("Config: {} turned on, failures will be ignored. Local store cleanup for job: {} resulted in exception: {}.", new Object[] { LocalStoreMonitorConfig.CONFIG_IGNORE_FAILURES, jobInstance, ex });
        }
    }
}
Also used : JobStatus(org.apache.samza.rest.model.JobStatus) Task(org.apache.samza.rest.model.Task) JobInstance(org.apache.samza.rest.proxy.job.JobInstance) TaskName(org.apache.samza.container.TaskName) File(java.io.File) IOException(java.io.IOException)

Aggregations

JobInstance (org.apache.samza.rest.proxy.job.JobInstance)6 File (java.io.File)4 JobStatus (org.apache.samza.rest.model.JobStatus)2 Task (org.apache.samza.rest.model.Task)2 IOException (java.io.IOException)1 URI (java.net.URI)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 LinkedHashSet (java.util.LinkedHashSet)1 PUT (javax.ws.rs.PUT)1 Path (javax.ws.rs.Path)1 Produces (javax.ws.rs.Produces)1 SamzaException (org.apache.samza.SamzaException)1 Config (org.apache.samza.config.Config)1 JobConfig (org.apache.samza.config.JobConfig)1 MapConfig (org.apache.samza.config.MapConfig)1 TaskName (org.apache.samza.container.TaskName)1 Job (org.apache.samza.rest.model.Job)1 Test (org.junit.Test)1