Examples with JobId com.spotify.helios.common.descriptors.JobId used on opensource projects

Example 56 with JobId

use of com.spotify.helios.common.descriptors.JobId in project helios by spotify.

the class RollingUpdateCommand method runWithJobId.

@Override
protected int runWithJobId(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final JobId jobId, final BufferedReader stdin) throws ExecutionException, InterruptedException, IOException {
    final String name = options.getString(nameArg.getDest());
    final long timeout = options.getLong(timeoutArg.getDest());
    final int parallelism = options.getInt(parallelismArg.getDest());
    final boolean async = options.getBoolean(asyncArg.getDest());
    final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest());
    final boolean migrate = options.getBoolean(migrateArg.getDest());
    final boolean overlap = options.getBoolean(overlapArg.getDest());
    final String token = options.getString(tokenArg.getDest());
    checkArgument(timeout > 0, "Timeout must be greater than 0");
    checkArgument(parallelism > 0, "Parallelism must be greater than 0");
    checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0");
    final long startTime = timeSupplier.get();
    final RolloutOptions rolloutOptions = RolloutOptions.newBuilder().setTimeout(timeout).setParallelism(parallelism).setMigrate(migrate).setOverlap(overlap).setToken(token).build();
    final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get();
    if (response.getStatus() != RollingUpdateResponse.Status.OK) {
        if (!json) {
            out.println("Failed: " + response);
        } else {
            out.println(response.toJsonString());
        }
        return 1;
    }
    if (!json) {
        out.println(format("Rolling update%s started: %s -> %s " + "(parallelism=%d, timeout=%d, overlap=%b, token=%s)%s", async ? " (async)" : "", name, jobId.toShortString(), parallelism, timeout, overlap, token, async ? "" : "\n"));
    }
    final Map<String, Object> jsonOutput = Maps.newHashMap();
    jsonOutput.put("parallelism", parallelism);
    jsonOutput.put("timeout", timeout);
    jsonOutput.put("overlap", overlap);
    jsonOutput.put("token", token);
    if (async) {
        if (json) {
            jsonOutput.put("status", response.getStatus());
            out.println(Json.asStringUnchecked(jsonOutput));
        }
        return 0;
    }
    String error = "";
    boolean failed = false;
    boolean timedOut = false;
    final Set<String> reported = Sets.newHashSet();
    while (true) {
        final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get();
        if (status == null) {
            failed = true;
            error = "Failed to fetch deployment-group status";
            break;
        }
        if (!jobId.equals(status.getDeploymentGroup().getJobId())) {
            // Another rolling-update was started, overriding this one -- exit
            failed = true;
            error = "Deployment-group job id changed during rolling-update";
            break;
        }
        if (!json) {
            for (final DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) {
                final JobId hostJobId = hostStatus.getJobId();
                final String host = hostStatus.getHost();
                final TaskStatus.State state = hostStatus.getState();
                final boolean done = hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING;
                if (done && reported.add(host)) {
                    out.println(format("%s -> %s (%d/%d)", host, state, reported.size(), status.getHostStatuses().size()));
                }
            }
        }
        if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) {
            if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) {
                failed = true;
                error = status.getError();
            }
            break;
        }
        if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) {
            // Rollout timed out
            timedOut = true;
            break;
        }
        sleepFunction.sleep(POLL_INTERVAL_MILLIS);
    }
    final double duration = (timeSupplier.get() - startTime) / 1000.0;
    if (json) {
        if (failed) {
            jsonOutput.put("status", "FAILED");
            jsonOutput.put("error", error);
        } else if (timedOut) {
            jsonOutput.put("status", "TIMEOUT");
        } else {
            jsonOutput.put("status", "DONE");
        }
        jsonOutput.put("duration", duration);
        out.println(Json.asStringUnchecked(jsonOutput));
    } else {
        out.println();
        if (failed) {
            out.println(format("Failed: %s", error));
        } else if (timedOut) {
            out.println("Timed out! (rolling-update still in progress)");
        } else {
            out.println("Done.");
        }
        out.println(format("Duration: %.2f s", duration));
    }
    return (failed || timedOut) ? 1 : 0;
}

Example 57 with JobId

use of com.spotify.helios.common.descriptors.JobId in project helios by spotify.

the class HeliosSoloDeploymentTest method testUndeployLeftoverJobs.

@Test
public void testUndeployLeftoverJobs() throws Exception {
    final HeliosSoloDeployment solo = buildHeliosSoloDeployment();
    final ListenableFuture<List<String>> hostsFuture = Futures.<List<String>>immediateFuture(ImmutableList.of(HOST1, HOST2));
    when(heliosClient.listHosts()).thenReturn(hostsFuture);
    // These futures represent HostStatuses when the job is still deployed
    final ListenableFuture<HostStatus> statusFuture11 = Futures.immediateFuture(HostStatus.newBuilder().setStatus(Status.UP).setStatuses(ImmutableMap.of(JOB_ID1, TASK_STATUS1)).setJobs(ImmutableMap.of(JOB_ID1, Deployment.of(JOB_ID1, Goal.START))).build());
    final ListenableFuture<HostStatus> statusFuture21 = Futures.immediateFuture(HostStatus.newBuilder().setStatus(Status.UP).setStatuses(ImmutableMap.of(JOB_ID2, TASK_STATUS2)).setJobs(ImmutableMap.of(JOB_ID2, Deployment.of(JOB_ID2, Goal.START))).build());
    // These futures represent HostStatuses when the job is undeployed
    final ListenableFuture<HostStatus> statusFuture12 = Futures.immediateFuture(HostStatus.newBuilder().setStatus(Status.UP).setStatuses(Collections.<JobId, TaskStatus>emptyMap()).setJobs(ImmutableMap.of(JOB_ID1, Deployment.of(JOB_ID1, Goal.START))).build());
    final ListenableFuture<HostStatus> statusFuture22 = Futures.immediateFuture(HostStatus.newBuilder().setStatus(Status.UP).setStatuses(Collections.<JobId, TaskStatus>emptyMap()).setJobs(ImmutableMap.of(JOB_ID2, Deployment.of(JOB_ID2, Goal.START))).build());
    //noinspection unchecked
    when(heliosClient.hostStatus(HOST1)).thenReturn(statusFuture11);
    //noinspection unchecked
    when(heliosClient.hostStatus(HOST2)).thenReturn(statusFuture21);
    final ListenableFuture<JobUndeployResponse> undeployFuture1 = Futures.immediateFuture(new JobUndeployResponse(JobUndeployResponse.Status.OK, HOST1, JOB_ID1));
    final ListenableFuture<JobUndeployResponse> undeployFuture2 = Futures.immediateFuture(new JobUndeployResponse(JobUndeployResponse.Status.OK, HOST2, JOB_ID2));
    // when undeploy is called, respond correctly & patch the mock to return
    // the undeployed HostStatus
    when(heliosClient.undeploy(JOB_ID1, HOST1)).thenAnswer(new Answer<ListenableFuture<JobUndeployResponse>>() {

        @Override
        public ListenableFuture<JobUndeployResponse> answer(final InvocationOnMock invocation) throws Throwable {
            when(heliosClient.hostStatus(HOST1)).thenReturn(statusFuture12);
            return undeployFuture1;
        }
    });
    when(heliosClient.undeploy(JOB_ID2, HOST2)).thenAnswer(new Answer<ListenableFuture<JobUndeployResponse>>() {

        @Override
        public ListenableFuture<JobUndeployResponse> answer(final InvocationOnMock invocation) throws Throwable {
            when(heliosClient.hostStatus(HOST1)).thenReturn(statusFuture22);
            return undeployFuture2;
        }
    });
    solo.undeployLeftoverJobs();
    verify(heliosClient).undeploy(JOB_ID1, HOST1);
    verify(heliosClient).undeploy(JOB_ID2, HOST2);
}

Example 58 with JobId

use of com.spotify.helios.common.descriptors.JobId in project helios by spotify.

the class JobNamePrefixTest method testJobNamePrefix.

@Test
public void testJobNamePrefix() throws Exception {
    // Create four jobs which represent these use cases:
    //  job1 - Created, deployed, locked. Simulates a job being used by another process. The
    //         job should not get undeployed or deleted since it is in use.
    //  job2 - Created, not deployed, locked. Simulates a job being used by another process. The
    //         job should not get deleted since it is in use.
    //  job3 - Created, deployed, not locked. Simulates an old job no longer in use, which should
    //         be undeployed and deleted.
    //  job4 - Created, not deployed, not locked. Simulates an old job no longer in use, which
    //         should be deleted.
    // job1 - create and deploy
    final JobId jobId1 = createJob(testJobName + "_1", testJobVersion, BUSYBOX, IDLE_COMMAND);
    deployJob(jobId1, testHost1);
    // job2 - create
    final JobId jobId2 = createJob(testJobName + "_2", testJobVersion, BUSYBOX, IDLE_COMMAND);
    // job3 - create and deploy
    final JobId jobId3 = createJob(testJobName + "_3", testJobVersion, BUSYBOX, IDLE_COMMAND);
    deployJob(jobId3, testHost1);
    // job4 - create
    final JobId jobId4 = createJob(testJobName + "_4", testJobVersion, BUSYBOX, IDLE_COMMAND);
    try (// Create prefix files for all four jobs. They will be locked by default.
    JobPrefixFile file1 = JobPrefixFile.create(jobId1.getName(), prefixDirectory);
        JobPrefixFile file2 = JobPrefixFile.create(jobId2.getName(), prefixDirectory);
        JobPrefixFile file3 = JobPrefixFile.create(jobId3.getName(), prefixDirectory);
        JobPrefixFile file4 = JobPrefixFile.create(jobId4.getName(), prefixDirectory)) {
        // Release the locks of jobs 3 and 4 so they can be cleaned up
        file3.release();
        file4.release();
        assertThat(testResult(JobNamePrefixTestImpl.class), isSuccessful());
        final Map<JobId, Job> jobs = client.jobs().get();
        // Verify job1 is still deployed and the prefix file has not been deleted.
        assertThat(jobs, hasKey(jobId1));
        final JobStatus status1 = client.jobStatus(jobId1).get();
        assertThat(status1.getDeployments().size(), is(1));
        assertTrue(fileExists(prefixDirectory, jobId1.getName()));
        // Verify job2 still exists, is not deployed, and the prefix file is still there.
        assertThat(jobs, hasKey(jobId2));
        final JobStatus status2 = client.jobStatus(jobId2).get();
        assertThat(status2.getDeployments().size(), is(0));
        assertTrue(fileExists(prefixDirectory, jobId2.getName()));
        // Verify that job3 has been deleted (which means it has also been undeployed), and
        // the prefix file has been deleted.
        assertThat(jobs, not(hasKey(jobId3)));
        assertFalse(fileExists(prefixDirectory, jobId3.getName()));
        // Verify that job4 and its prefix file have been deleted.
        assertThat(jobs, not(hasKey(jobId4)));
        assertFalse(fileExists(prefixDirectory, jobId4.getName()));
        // Verify the prefix file created during the run of JobNamePrefixTest was deleted
        assertFalse(fileExists(prefixDirectory, jobPrefixFile.prefix()));
    }
}

Example 59 with JobId

use of com.spotify.helios.common.descriptors.JobId in project helios by spotify.

the class JobStatusCommand method run.

@Override
int run(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final BufferedReader stdin) throws ExecutionException, InterruptedException {
    final String jobIdString = options.getString(jobArg.getDest());
    final String hostPattern = options.getString(hostArg.getDest());
    final boolean full = options.getBoolean(fullArg.getDest());
    final Map<JobId, Job> jobs;
    if (Strings.isNullOrEmpty(jobIdString)) {
        jobs = client.jobs().get();
    } else {
        jobs = client.jobs(jobIdString).get();
    }
    if (jobs == null) {
        out.printf("The specified Helios master either returned an error or job id matcher " + "\"%s\" matched no jobs%n", jobIdString);
        return 1;
    }
    final Set<JobId> jobIds = jobs.keySet();
    if (!Strings.isNullOrEmpty(jobIdString) && jobIds.isEmpty()) {
        if (json) {
            out.println("{ }");
        } else {
            out.printf("job id matcher \"%s\" matched no jobs%n", jobIdString);
        }
        return 1;
    }
    final Map<JobId, JobStatus> statuses = Maps.newTreeMap();
    statuses.putAll(client.jobStatuses(jobIds).get());
    if (json) {
        showJsonStatuses(out, hostPattern, jobIds, statuses);
        return 0;
    }
    final JobStatusTable table = jobStatusTable(out, full);
    final boolean noHostMatchedEver = showStatusesForHosts(hostPattern, jobIds, statuses, new HostStatusDisplayer() {

        @Override
        public void matchedStatus(JobStatus jobStatus, Iterable<String> matchingHosts, Map<String, TaskStatus> taskStatuses) {
            displayTask(full, table, jobStatus.getJob().getId(), jobStatus, taskStatuses, matchingHosts);
        }
    });
    if (noHostMatchedEver) {
        String domainsSwitchString = "";
        final List<String> domains = options.get("domains");
        if (domains.size() > 0) {
            domainsSwitchString = "-d " + Joiner.on(",").join(domains);
        }
        out.printf("There are no jobs deployed to hosts with the host pattern '%s'%n" + "Run 'helios %s hosts %s' to check your host exists and is up.%n", hostPattern, domainsSwitchString, hostPattern);
        return 1;
    }
    table.print();
    return 0;
}

Example 60 with JobId

use of com.spotify.helios.common.descriptors.JobId in project helios by spotify.

the class JobStatusCommand method showStatusesForHosts.

private boolean showStatusesForHosts(final String hostPattern, final Set<JobId> jobIds, final Map<JobId, JobStatus> statuses, final HostStatusDisplayer statusDisplayer) {
    boolean noHostMatchedEver = true;
    for (final JobId jobId : Ordering.natural().sortedCopy(jobIds)) {
        final JobStatus jobStatus = statuses.get(jobId);
        // jobStatus will be null if the job was deleted after we first got the list of job IDs
        if (jobStatus == null) {
            continue;
        }
        final Map<String, TaskStatus> taskStatuses = Maps.newTreeMap();
        taskStatuses.putAll(jobStatus.getTaskStatuses());
        // This will help us see hosts where jobs aren't running correctly.
        for (final String host : jobStatus.getDeployments().keySet()) {
            if (!taskStatuses.containsKey(host)) {
                taskStatuses.put(host, null);
            }
        }
        final FluentIterable<String> matchingHosts = FluentIterable.from(taskStatuses.keySet()).filter(containsPattern(hostPattern));
        if (Strings.isNullOrEmpty(hostPattern) || !Strings.isNullOrEmpty(hostPattern) && !matchingHosts.isEmpty()) {
            noHostMatchedEver = false;
        }
        statusDisplayer.matchedStatus(jobStatus, matchingHosts, taskStatuses);
    }
    return noHostMatchedEver;
}