use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class TemporaryJobs method removeOldJobs.
/**
* Undeploys and deletes jobs leftover from previous runs of TemporaryJobs. This would happen if
* the test was terminated before the cleanup code was called. This method will iterate over each
* file in the specified directory. Each filename is the prefix that was used for job names
* during previous runs. The method will undeploy and delete any jobs that have a matching
* prefix, and the delete the file. If the file is locked, it is currently in use, and will be
* skipped.
*/
private void removeOldJobs() throws ExecutionException, InterruptedException, IOException {
// is used as a @Rule in a test class with many test methods
if (removedOldJobs) {
return;
}
final File[] files = prefixDirectory.toFile().listFiles();
if (files == null || files.length == 0) {
return;
}
log.info("Removing old temporary jobs");
final Map<JobId, Job> jobs = client.jobs().get();
// Iterate over all files in the directory
for (final File file : files) {
// directories. We don't expect any, but skip them just in case.
if (file.getName().endsWith(".tmp") || file.isDirectory()) {
continue;
}
// used by another process. In either case, skip over it.
try (JobPrefixFile prefixFile = JobPrefixFile.tryFromExistingFile(file.toPath())) {
if (prefixFile == null) {
log.debug("Unable to create JobPrefixFile for {}", file.getPath());
continue;
}
boolean jobRemovalFailed = false;
// Iterate over jobs, looking for ones with a matching prefix.
for (final Map.Entry<JobId, Job> entry : jobs.entrySet()) {
final JobId jobId = entry.getKey();
// Skip over job if the id doesn't start with current filename.
if (!jobId.getName().startsWith(prefixFile.prefix())) {
continue;
}
// Get list of all hosts where this job is deployed, and undeploy
final JobStatus status = client.jobStatus(entry.getKey()).get();
final List<String> hosts = ImmutableList.copyOf(status.getDeployments().keySet());
final List<AssertionError> errors = undeploy(client, entry.getValue(), hosts, new ArrayList<AssertionError>());
// Set flag indicating if any errors occur
if (!errors.isEmpty()) {
jobRemovalFailed = true;
}
}
// leave it there so we can try again next time.
if (!jobRemovalFailed) {
prefixFile.delete();
}
} catch (NoSuchFileException e) {
log.debug("File {} already processed by somebody else.", file.getPath());
} catch (Exception e) {
// log exception and continue on to next file
log.warn("Exception processing file {}", file.getPath(), e);
}
}
removedOldJobs = true;
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class JobNamePrefixTest method testJobNamePrefix.
@Test
public void testJobNamePrefix() throws Exception {
// Create four jobs which represent these use cases:
// job1 - Created, deployed, locked. Simulates a job being used by another process. The
// job should not get undeployed or deleted since it is in use.
// job2 - Created, not deployed, locked. Simulates a job being used by another process. The
// job should not get deleted since it is in use.
// job3 - Created, deployed, not locked. Simulates an old job no longer in use, which should
// be undeployed and deleted.
// job4 - Created, not deployed, not locked. Simulates an old job no longer in use, which
// should be deleted.
// job1 - create and deploy
final JobId jobId1 = createJob(testJobName + "_1", testJobVersion, BUSYBOX, IDLE_COMMAND);
deployJob(jobId1, testHost1);
// job2 - create
final JobId jobId2 = createJob(testJobName + "_2", testJobVersion, BUSYBOX, IDLE_COMMAND);
// job3 - create and deploy
final JobId jobId3 = createJob(testJobName + "_3", testJobVersion, BUSYBOX, IDLE_COMMAND);
deployJob(jobId3, testHost1);
// job4 - create
final JobId jobId4 = createJob(testJobName + "_4", testJobVersion, BUSYBOX, IDLE_COMMAND);
try (// Create prefix files for all four jobs. They will be locked by default.
JobPrefixFile file1 = JobPrefixFile.create(jobId1.getName(), prefixDirectory);
JobPrefixFile file2 = JobPrefixFile.create(jobId2.getName(), prefixDirectory);
JobPrefixFile file3 = JobPrefixFile.create(jobId3.getName(), prefixDirectory);
JobPrefixFile file4 = JobPrefixFile.create(jobId4.getName(), prefixDirectory)) {
// Release the locks of jobs 3 and 4 so they can be cleaned up
file3.release();
file4.release();
assertThat(testResult(JobNamePrefixTestImpl.class), isSuccessful());
final Map<JobId, Job> jobs = client.jobs().get();
// Verify job1 is still deployed and the prefix file has not been deleted.
assertThat(jobs, hasKey(jobId1));
final JobStatus status1 = client.jobStatus(jobId1).get();
assertThat(status1.getDeployments().size(), is(1));
assertTrue(fileExists(prefixDirectory, jobId1.getName()));
// Verify job2 still exists, is not deployed, and the prefix file is still there.
assertThat(jobs, hasKey(jobId2));
final JobStatus status2 = client.jobStatus(jobId2).get();
assertThat(status2.getDeployments().size(), is(0));
assertTrue(fileExists(prefixDirectory, jobId2.getName()));
// Verify that job3 has been deleted (which means it has also been undeployed), and
// the prefix file has been deleted.
assertThat(jobs, not(hasKey(jobId3)));
assertFalse(fileExists(prefixDirectory, jobId3.getName()));
// Verify that job4 and its prefix file have been deleted.
assertThat(jobs, not(hasKey(jobId4)));
assertFalse(fileExists(prefixDirectory, jobId4.getName()));
// Verify the prefix file created during the run of JobNamePrefixTest was deleted
assertFalse(fileExists(prefixDirectory, jobPrefixFile.prefix()));
}
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class JobsTest method testGetJobDescription.
@Test
public void testGetJobDescription() {
final String image = "spotify/busybox:latest";
final Job job = Job.newBuilder().setImage(image).setName("testGetJobDescription").setVersion("1").build();
final String shortHash = job.getId().getHash().substring(0, 7);
// Simple test to verify the job description contains the image name and a shortened job hash.
assertThat(Jobs.getJobDescription(job), both(startsWith(image)).and(containsString(shortHash)));
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class JobStatusCommand method run.
@Override
int run(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final BufferedReader stdin) throws ExecutionException, InterruptedException {
final String jobIdString = options.getString(jobArg.getDest());
final String hostPattern = options.getString(hostArg.getDest());
final boolean full = options.getBoolean(fullArg.getDest());
final Map<JobId, Job> jobs;
if (Strings.isNullOrEmpty(jobIdString)) {
jobs = client.jobs().get();
} else {
jobs = client.jobs(jobIdString).get();
}
if (jobs == null) {
out.printf("The specified Helios master either returned an error or job id matcher " + "\"%s\" matched no jobs%n", jobIdString);
return 1;
}
final Set<JobId> jobIds = jobs.keySet();
if (!Strings.isNullOrEmpty(jobIdString) && jobIds.isEmpty()) {
if (json) {
out.println("{ }");
} else {
out.printf("job id matcher \"%s\" matched no jobs%n", jobIdString);
}
return 1;
}
final Map<JobId, JobStatus> statuses = Maps.newTreeMap();
statuses.putAll(client.jobStatuses(jobIds).get());
if (json) {
showJsonStatuses(out, hostPattern, jobIds, statuses);
return 0;
}
final JobStatusTable table = jobStatusTable(out, full);
final boolean noHostMatchedEver = showStatusesForHosts(hostPattern, jobIds, statuses, new HostStatusDisplayer() {
@Override
public void matchedStatus(JobStatus jobStatus, Iterable<String> matchingHosts, Map<String, TaskStatus> taskStatuses) {
displayTask(full, table, jobStatus.getJob().getId(), jobStatus, taskStatuses, matchingHosts);
}
});
if (noHostMatchedEver) {
String domainsSwitchString = "";
final List<String> domains = options.get("domains");
if (domains.size() > 0) {
domainsSwitchString = "-d " + Joiner.on(",").join(domains);
}
out.printf("There are no jobs deployed to hosts with the host pattern '%s'%n" + "Run 'helios %s hosts %s' to check your host exists and is up.%n", hostPattern, domainsSwitchString, hostPattern);
return 1;
}
table.print();
return 0;
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class ZooKeeperMasterModel method removeJob.
/**
* Deletes a job from ZooKeeper. Ensures that job is not currently running anywhere.
*/
@Override
public Job removeJob(final JobId id, final String token) throws JobDoesNotExistException, JobStillDeployedException, TokenVerificationException {
log.info("removing job: id={}", id);
final ZooKeeperClient client = provider.get("removeJob");
final Job job = getJob(client, id);
if (job == null) {
throw new JobDoesNotExistException(id);
}
verifyToken(token, job);
// TODO (dano): handle retry failures
try {
final ImmutableList.Builder<ZooKeeperOperation> operations = ImmutableList.builder();
final UUID jobCreationOperationId = getJobCreation(client, id);
if (jobCreationOperationId != null) {
operations.add(delete(Paths.configJobCreation(id, jobCreationOperationId)));
}
operations.add(delete(Paths.configJobHosts(id)), delete(Paths.configJobRefShort(id)), delete(Paths.configJob(id)), // change down the tree. Effectively, make it that version == cVersion.
set(Paths.configJobs(), UUID.randomUUID().toString().getBytes()));
client.transaction(operations.build());
} catch (final NoNodeException e) {
throw new JobDoesNotExistException(id);
} catch (final NotEmptyException e) {
throw new JobStillDeployedException(id, listJobHosts(client, id));
} catch (final KeeperException e) {
throw new HeliosRuntimeException("removing job " + id + " failed", e);
}
// Delete job history on a best effort basis
try {
client.deleteRecursive(Paths.historyJob(id));
} catch (NoNodeException ignored) {
// There's no history for this job
} catch (KeeperException e) {
log.warn("error removing job history for job {}", id, e);
}
return job;
}
Aggregations