Examples with JobStatus com.google.cloud.dataproc.v1beta2.JobStatus used on opensource projects

Example 1 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project pipelite by enasequence.

the class KubernetesExecutorTest method describeJobsStateSuccess.

@Test
public void describeJobsStateSuccess() {
    JobStatus jobStatus = new JobStatus();
    jobStatus.setCompletionTime("test");
    assertThat(KubernetesExecutor.describeJobsResultFromStatus(jobStatus).isSuccess()).isTrue();
    jobStatus = new JobStatus();
    JobCondition jobCondition = new JobCondition();
    jobCondition.setType("Complete");
    jobCondition.setStatus("true");
    jobStatus.setConditions(Arrays.asList(jobCondition));
    assertThat(KubernetesExecutor.describeJobsResultFromStatus(jobStatus).isSuccess()).isTrue();
}

Example 2 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project marduk by entur.

the class KubernetesJobRunner method runJob.

/**
 * Run a Kubernetes job
 *
 * @param cronJobName   name of the CronJob used as a template
 * @param jobNamePrefix prefix for the Kubernetes job name
 * @param envVars       environment variables to be provided to the job
 * @param timestamp     timestamp used to create a unique name for the Kubernetes job.
 */
public void runJob(String cronJobName, String jobNamePrefix, List<EnvVar> envVars, String timestamp) {
    try (final KubernetesClient kubernetesClient = new DefaultKubernetesClient()) {
        String jobName = jobNamePrefix + '-' + timestamp;
        final Job job = retrieveOrCreateJob(jobName, cronJobName, envVars, kubernetesClient);
        final CountDownLatch watchLatch = new CountDownLatch(1);
        MardukPodWatcher mardukPodWatcher = new MardukPodWatcher(job, watchLatch, jobName);
        try (Watch watch = kubernetesClient.pods().inNamespace(kubernetesNamespace).withLabel("job-name", jobName).watch(mardukPodWatcher)) {
            boolean jobCompletedBeforeTimeout = watchLatch.await(jobTimeoutSecond, TimeUnit.SECONDS);
            if (!jobCompletedBeforeTimeout) {
                throw new KubernetesJobRunnerException("Timeout while waiting for the Graph Builder job " + jobName + " to complete.");
            }
            JobStatus status = kubernetesClient.batch().v1().jobs().inNamespace(kubernetesNamespace).withName(jobName).get().getStatus();
            LOGGER.debug("Kubernetes Job status on completion: {}", status);
            // test the pod status rather than the job status since the job status may be out of sync with the pod status
            if (mardukPodWatcher.isSucceeded()) {
                LOGGER.info("The Graph Builder job {} completed successfully.", jobName);
            } else if (mardukPodWatcher.isKubernetesClientError()) {
                throw new KubernetesJobRunnerException("Kubernetes client error while watching the Graph Builder job " + jobName);
            } else {
                throw new KubernetesJobRunnerException("The Graph Builder job " + jobName + " failed.");
            }
        } catch (KubernetesClientException e) {
            throw new KubernetesJobRunnerException("Could not watch pod", e);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new KubernetesJobRunnerException("Interrupted while watching pod", e);
        } finally {
            // Delete job after completion unless there was a Kubernetes error that can be retried
            if (!mardukPodWatcher.isKubernetesClientError() && deleteJobAfterCompletion) {
                LOGGER.info("Deleting job {} after completion.", jobName);
                deleteKubernetesJob(kubernetesClient, job);
                LOGGER.info("Deleted job {} after completion.", jobName);
            }
        }
    }
}

Example 3 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class KubernetesV1DistributedService method connectToInstance.

@Override
default <S> S connectToInstance(AccountDeploymentDetails<KubernetesAccount> details, SpinnakerRuntimeSettings runtimeSettings, SpinnakerService<S> sidecar, String instanceId) {
    ServiceSettings settings = runtimeSettings.getServiceSettings(sidecar);
    String namespace = getNamespace(settings);
    int localPort = SocketUtils.findAvailableTcpPort();
    int targetPort = settings.getPort();
    List<String> command = KubernetesV1ProviderUtils.kubectlPortForwardCommand(details, namespace, instanceId, targetPort, localPort);
    JobRequest request = new JobRequest().setTokenizedCommand(command);
    String jobId = getJobExecutor().startJob(request);
    // Wait for the proxy to spin up.
    DaemonTaskHandler.safeSleep(TimeUnit.SECONDS.toMillis(5));
    JobStatus status = getJobExecutor().updateJob(jobId);
    // This should be a long-running job.
    if (status.getState() == JobStatus.State.COMPLETED) {
        throw new HalException(Problem.Severity.FATAL, "Unable to establish a proxy against " + getServiceName() + ":\n" + status.getStdOut() + "\n" + status.getStdErr());
    }
    return getServiceInterfaceFactory().createService(settings.getScheme() + "://localhost:" + localPort, sidecar);
}

Example 4 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class KubernetesV1ProviderUtils method storeInstanceLogs.

static void storeInstanceLogs(JobExecutor jobExecutor, AccountDeploymentDetails<KubernetesAccount> details, String namespace, String instanceName, String containerName, File outputFile) {
    List<String> command = kubectlAccountCommand(details);
    command.add("--namespace");
    command.add(namespace);
    command.add("logs");
    command.add(instanceName);
    command.add(containerName);
    JobRequest request = new JobRequest().setTokenizedCommand(command);
    JobStatus status;
    try {
        status = jobExecutor.backoffWait(jobExecutor.startJob(request));
    } catch (InterruptedException e) {
        throw new DaemonTaskInterrupted(e);
    }
    try {
        IOUtils.write(status.getStdOut().getBytes(), new FileOutputStream(new File(outputFile, containerName)));
    } catch (IOException e) {
        throw new HalException(Severity.FATAL, "Unable to store logs: " + e.getMessage(), e);
    }
}

Example 5 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class KubernetesV2Utils method apply.

public static void apply(KubernetesAccount account, String manifest) {
    manifest = prettify(manifest);
    List<String> command = kubectlPrefix(account);
    command.add("apply");
    command.add("-f");
    // read from stdin
    command.add("-");
    JobRequest request = new JobRequest().setTokenizedCommand(command);
    ByteArrayOutputStream stdout = new ByteArrayOutputStream();
    ByteArrayOutputStream stderr = new ByteArrayOutputStream();
    String jobId = DaemonTaskHandler.getJobExecutor().startJob(request, System.getenv(), new ByteArrayInputStream(manifest.getBytes()), stdout, stderr);
    JobStatus status;
    try {
        status = DaemonTaskHandler.getJobExecutor().backoffWait(jobId);
    } catch (InterruptedException e) {
        throw new DaemonTaskInterrupted(e);
    }
    if (status.getState() != JobStatus.State.COMPLETED) {
        throw new HalException(Problem.Severity.FATAL, String.join("\n", "Unterminated deployment of manifest:", manifest, stderr.toString(), stdout.toString()));
    }
    if (status.getResult() != JobStatus.Result.SUCCESS) {
        throw new HalException(Problem.Severity.FATAL, String.join("\n", "Failed to deploy manifest:", manifest, stderr.toString(), stdout.toString()));
    }
}