Examples with Job com.spotify.helios.common.descriptors.Job used on opensource projects

Example 41 with Job

use of com.spotify.helios.common.descriptors.Job in project helios by spotify.

the class ZooKeeperMasterModel method getInitRollingUpdateOps.

private RollingUpdateOp getInitRollingUpdateOps(final DeploymentGroup deploymentGroup, final List<String> updateHosts, final List<String> undeployHosts, final ZooKeeperClient zooKeeperClient) throws KeeperException {
    final List<RolloutTask> rolloutTasks = new ArrayList<>();
    // give precedence to the updateHosts list so we don't end up in a state where we updated a host
    // and then removed the job from it (because of buggy logic in the calling method)
    final List<String> updateHostsCopy = new ArrayList<>(updateHosts);
    final List<String> undeployHostsCopy = new ArrayList<>(undeployHosts);
    undeployHostsCopy.removeAll(updateHostsCopy);
    // we only care about hosts that are UP
    final List<String> upHostsToUndeploy = undeployHostsCopy.stream().filter(host -> checkHostUp(zooKeeperClient, host)).collect(Collectors.toList());
    final List<String> upHostsToDeploy = updateHostsCopy.stream().filter(host -> checkHostUp(zooKeeperClient, host)).collect(Collectors.toList());
    rolloutTasks.addAll(RollingUndeployPlanner.of(deploymentGroup).plan(upHostsToUndeploy));
    rolloutTasks.addAll(RollingUpdatePlanner.of(deploymentGroup).plan(upHostsToDeploy));
    log.info("generated rolloutTasks for deployment-group name={} " + "updateHosts={} undeployHosts={}: {}", deploymentGroup.getName(), updateHosts, undeployHosts, rolloutTasks);
    final DeploymentGroupTasks tasks = DeploymentGroupTasks.newBuilder().setRolloutTasks(rolloutTasks).setTaskIndex(0).setDeploymentGroup(deploymentGroup).build();
    return new RollingUpdateOpFactory(tasks, DEPLOYMENT_GROUP_EVENT_FACTORY).start(deploymentGroup, zooKeeperClient);
}

Example 42 with Job

use of com.spotify.helios.common.descriptors.Job in project helios by spotify.

the class JobsResource method post.

/**
   * Create a job given the definition in {@code job}.
   *
   * @param job The job to create.
   * @param username The user creating the job.
   * @return The response.
   */
@POST
@Produces(APPLICATION_JSON)
@Timed
@ExceptionMetered
public CreateJobResponse post(@Valid final Job job, @RequestUser final String username) {
    final Job.Builder clone = job.toBuilder().setCreatingUser(username).setCreated(clock.now().getMillis()).setHash(job.getId().getHash());
    final Job actualJob = clone.build();
    final Collection<String> errors = jobValidator.validate(actualJob);
    final String jobIdString = actualJob.getId().toString();
    if (!errors.isEmpty()) {
        throw badRequest(new CreateJobResponse(INVALID_JOB_DEFINITION, ImmutableList.copyOf(errors), jobIdString));
    }
    try {
        model.addJob(actualJob);
    } catch (JobExistsException e) {
        throw badRequest(new CreateJobResponse(JOB_ALREADY_EXISTS, ImmutableList.<String>of(), jobIdString));
    }
    log.info("created job: {}", actualJob);
    return new CreateJobResponse(CreateJobResponse.Status.OK, ImmutableList.<String>of(), jobIdString);
}

Example 43 with Job

use of com.spotify.helios.common.descriptors.Job in project helios by spotify.

the class ExpiredJobReaper method runOneIteration.

@Override
public void runOneIteration() {
    for (final Entry<JobId, Job> entry : masterModel.getJobs().entrySet()) {
        final JobId jobId = entry.getKey();
        final Job job = entry.getValue();
        if (job.getExpires() == null) {
            //noinspection UnnecessaryContinue
            continue;
        } else if (job.getExpires().getTime() <= clock.now().getMillis()) {
            final JobStatus status = masterModel.getJobStatus(jobId);
            final List<String> hosts = ImmutableList.copyOf(status.getDeployments().keySet());
            for (final String host : hosts) {
                try {
                    masterModel.undeployJob(host, jobId, job.getToken());
                } catch (HostNotFoundException e) {
                    log.error("couldn't undeploy job {} from host {} when it hit deadline", jobId, host, e);
                } catch (JobNotDeployedException e) {
                    log.debug("job {} was already undeployed when it hit deadline", jobId, e);
                } catch (TokenVerificationException e) {
                    log.error("couldn't undeploy job {} from host {} because token verification failed", jobId, host, e);
                }
            }
            try {
                masterModel.removeJob(jobId, job.getToken());
            } catch (JobDoesNotExistException e) {
                log.debug("job {} was already removed when it hit deadline", jobId, e);
            } catch (JobStillDeployedException e) {
                log.debug("job {} still deployed on some host(s) after expiry reap", jobId, e);
            } catch (TokenVerificationException e) {
                log.error("couldn't remove job {} because token verification failed", jobId, e);
            }
        }
    }
}

Example 44 with Job

use of com.spotify.helios.common.descriptors.Job in project helios by spotify.

the class JobHistoryReaper method processItem.

@Override
void processItem(final String jobId) {
    log.info("Deciding whether to reap job history for job {}", jobId);
    final JobId id = JobId.fromString(jobId);
    final Job job = masterModel.getJob(id);
    if (job == null) {
        try {
            client.deleteRecursive(Paths.historyJob(id));
            log.info("Reaped job history for job {}", jobId);
        } catch (NoNodeException ignored) {
        // Something deleted the history right before we got to it. Ignore and keep going.
        } catch (KeeperException e) {
            log.warn("error reaping job history for job {}", jobId, e);
        }
    }
}

Example 45 with Job

use of com.spotify.helios.common.descriptors.Job in project helios by spotify.

the class ZooKeeperMasterModel method getJobs.

/**
   * Returns a {@link Map} of {@link JobId} to {@link Job} objects for all of the jobs known.
   */
@Override
public Map<JobId, Job> getJobs() {
    log.debug("getting jobs");
    final String folder = Paths.configJobs();
    final ZooKeeperClient client = provider.get("getJobs");
    try {
        final List<String> ids;
        try {
            ids = client.getChildren(folder);
        } catch (NoNodeException e) {
            return Maps.newHashMap();
        }
        final Map<JobId, Job> descriptors = Maps.newHashMap();
        for (final String id : ids) {
            final JobId jobId = JobId.fromString(id);
            final String path = Paths.configJob(jobId);
            try {
                final byte[] data = client.getData(path);
                final Job descriptor = parse(data, Job.class);
                descriptors.put(descriptor.getId(), descriptor);
            } catch (NoNodeException e) {
                // Ignore, the job was deleted before we had a chance to read it.
                log.debug("Ignoring deleted job {}", jobId);
            }
        }
        return descriptors;
    } catch (KeeperException | IOException e) {
        throw new HeliosRuntimeException("getting jobs failed", e);
    }
}