Examples with Job com.google.cloud.dataproc.v1.Job used on opensource projects

Example 26 with Job

use of com.google.cloud.dataproc.v1.Job in project stackgres by ongres.

the class DbOpsJob method createJob.

@Override
public Job createJob(StackGresDbOpsContext context) {
    final StackGresDbOps dbOps = context.getSource();
    final String retries = String.valueOf(DbOpsUtil.getCurrentRetry(dbOps));
    List<EnvVar> runEnvVars = getRunEnvVars(context);
    List<EnvVar> setResultEnvVars = getSetResultEnvVars(context);
    final String namespace = dbOps.getMetadata().getNamespace();
    final String name = dbOps.getMetadata().getName();
    final Map<String, String> labels = dbOpsLabelFactory.dbOpsPodLabels(context.getSource());
    final String timeout = DbOpsUtil.getTimeout(dbOps);
    return new JobBuilder().withNewMetadata().withNamespace(namespace).withName(jobName(dbOps)).withLabels(labels).endMetadata().withNewSpec().withBackoffLimit(0).withCompletions(1).withParallelism(1).withNewTemplate().withNewMetadata().withNamespace(namespace).withName(jobName(dbOps)).withLabels(labels).endMetadata().withNewSpec().withSecurityContext(podSecurityFactory.createResource(context)).withRestartPolicy("Never").withServiceAccountName(DbOpsRole.roleName(context)).withInitContainers(new ContainerBuilder().withName("set-dbops-running").withImage(getSetResultImage()).withImagePullPolicy("IfNotPresent").withEnv(ImmutableList.<EnvVar>builder().addAll(clusterEnvironmentVariables.listResources(context)).add(new EnvVarBuilder().withName("OP_NAME").withValue(dbOps.getSpec().getOp()).build(), new EnvVarBuilder().withName("NORMALIZED_OP_NAME").withValue(UPPERCASE_PATTERN.matcher(dbOps.getSpec().getOp()).replaceAll(result -> " " + result.group(1).toLowerCase(Locale.US))).build(), new EnvVarBuilder().withName("KEBAB_OP_NAME").withValue(UPPERCASE_PATTERN.matcher(dbOps.getSpec().getOp()).replaceAll(result -> "-" + result.group(1).toLowerCase(Locale.US))).build(), new EnvVarBuilder().withName("CLUSTER_NAMESPACE").withValue(namespace).build(), new EnvVarBuilder().withName("DB_OPS_NAME").withValue(name).build(), new EnvVarBuilder().withName("DB_OPS_CRD_NAME").withValue(CustomResource.getCRDName(StackGresDbOps.class)).build(), new EnvVarBuilder().withName("CURRENT_RETRY").withValue(retries).build()).addAll(Seq.of(DbOpsStatusCondition.values()).map(c -> new EnvVarBuilder().withName("CONDITION_" + c.name()).withValue(conditions.get(c)).build()).toList()).build()).withCommand("/bin/sh", "-ex", ClusterStatefulSetPath.LOCAL_BIN_SET_DBOPS_RUNNING_SH_PATH.path()).withVolumeMounts(ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(ClusterStatefulSetPath.LOCAL_BIN_SET_DBOPS_RUNNING_SH_PATH.filename()).withMountPath(ClusterStatefulSetPath.LOCAL_BIN_SET_DBOPS_RUNNING_SH_PATH.path()).withReadOnly(true)), ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(ClusterStatefulSetPath.LOCAL_BIN_SHELL_UTILS_PATH.filename()).withMountPath(ClusterStatefulSetPath.LOCAL_BIN_SHELL_UTILS_PATH.path()).withReadOnly(true))).build()).withContainers(new ContainerBuilder().withName("run-dbops").withImage(getRunImage(context)).withImagePullPolicy("IfNotPresent").withEnv(ImmutableList.<EnvVar>builder().addAll(clusterEnvironmentVariables.listResources(context)).add(new EnvVarBuilder().withName("OP_NAME").withValue(dbOps.getSpec().getOp()).build(), new EnvVarBuilder().withName("EXCLUSIVE_OP").withValue(String.valueOf(isExclusiveOp())).build(), new EnvVarBuilder().withName("NORMALIZED_OP_NAME").withValue(UPPERCASE_PATTERN.matcher(dbOps.getSpec().getOp()).replaceAll(result -> " " + result.group(1).toLowerCase(Locale.US))).build(), new EnvVarBuilder().withName("KEBAB_OP_NAME").withValue(UPPERCASE_PATTERN.matcher(dbOps.getSpec().getOp()).replaceAll(result -> "-" + result.group(1).toLowerCase(Locale.US))).build(), new EnvVarBuilder().withName("RUN_SCRIPT_PATH").withValue(Optional.ofNullable(getRunScript()).map(ClusterStatefulSetPath::path).orElse("")).build(), new EnvVarBuilder().withName("TIMEOUT").withValue(timeout).build()).addAll(runEnvVars).build()).withCommand("/bin/sh", "-ex", ClusterStatefulSetPath.LOCAL_BIN_RUN_DBOPS_SH_PATH.path()).withVolumeMounts(ClusterStatefulSetVolumeConfig.SHARED.volumeMount(context), ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(ClusterStatefulSetPath.LOCAL_BIN_RUN_DBOPS_SH_PATH.filename()).withMountPath(ClusterStatefulSetPath.LOCAL_BIN_RUN_DBOPS_SH_PATH.path()).withReadOnly(true)), ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(ClusterStatefulSetPath.LOCAL_BIN_SHELL_UTILS_PATH.filename()).withMountPath(ClusterStatefulSetPath.LOCAL_BIN_SHELL_UTILS_PATH.path()).withReadOnly(true)), ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(getRunScript().filename()).withMountPath(getRunScript().path()).withReadOnly(true))).build(), new ContainerBuilder().withName("set-dbops-result").withImage(StackGresComponent.KUBECTL.findLatestImageName()).withImagePullPolicy("IfNotPresent").withEnv(ImmutableList.<EnvVar>builder().addAll(clusterEnvironmentVariables.listResources(context)).add(new EnvVarBuilder().withName("OP_NAME").withValue(dbOps.getSpec().getOp()).build(), new EnvVarBuilder().withName("NORMALIZED_OP_NAME").withValue(UPPERCASE_PATTERN.matcher(dbOps.getSpec().getOp()).replaceAll(result -> " " + result.group(1).toLowerCase(Locale.US))).build(), new EnvVarBuilder().withName("KEBAB_OP_NAME").withValue(UPPERCASE_PATTERN.matcher(dbOps.getSpec().getOp()).replaceAll(result -> "-" + result.group(1).toLowerCase(Locale.US))).build(), new EnvVarBuilder().withName("SET_RESULT_SCRIPT_PATH").withValue(Optional.ofNullable(getSetResultScript()).map(ClusterStatefulSetPath::path).orElse("")).build(), new EnvVarBuilder().withName("CLUSTER_NAMESPACE").withValue(namespace).build(), new EnvVarBuilder().withName("DB_OPS_NAME").withValue(name).build(), new EnvVarBuilder().withName("DB_OPS_CRD_NAME").withValue(CustomResource.getCRDName(StackGresDbOps.class)).build(), new EnvVarBuilder().withName("JOB_POD_LABELS").withValue(Seq.seq(labels).append(Tuple.tuple("job-name", jobName(dbOps))).map(t -> t.v1 + "=" + t.v2).toString(",")).build()).addAll(Seq.of(DbOpsStatusCondition.values()).map(c -> new EnvVarBuilder().withName("CONDITION_" + c.name()).withValue(conditions.get(c)).build()).toList()).addAll(setResultEnvVars).build()).withCommand("/bin/sh", "-ex", ClusterStatefulSetPath.LOCAL_BIN_SET_DBOPS_RESULT_SH_PATH.path()).withVolumeMounts(ClusterStatefulSetVolumeConfig.SHARED.volumeMount(context), ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(ClusterStatefulSetPath.LOCAL_BIN_SET_DBOPS_RESULT_SH_PATH.filename()).withMountPath(ClusterStatefulSetPath.LOCAL_BIN_SET_DBOPS_RESULT_SH_PATH.path()).withReadOnly(true)), ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(ClusterStatefulSetPath.LOCAL_BIN_SHELL_UTILS_PATH.filename()).withMountPath(ClusterStatefulSetPath.LOCAL_BIN_SHELL_UTILS_PATH.path()).withReadOnly(true))).addAllToVolumeMounts(Optional.ofNullable(getSetResultScript()).map(script -> ClusterStatefulSetVolumeConfig.TEMPLATES.volumeMount(context, volumeMountBuilder -> volumeMountBuilder.withSubPath(script.filename()).withMountPath(script.path()).withReadOnly(true))).stream().collect(Collectors.toList())).build()).withVolumes(ClusterStatefulSetVolumeConfig.SHARED.volume(context), new VolumeBuilder(ClusterStatefulSetVolumeConfig.TEMPLATES.volume(context)).editConfigMap().withDefaultMode(// NOPMD
    0555).endConfigMap().build()).endSpec().endTemplate().endSpec().build();
}

Example 27 with Job

use of com.google.cloud.dataproc.v1.Job in project marduk by entur.

the class KubernetesJobRunner method runJob.

/**
 * Run a Kubernetes job
 *
 * @param cronJobName   name of the CronJob used as a template
 * @param jobNamePrefix prefix for the Kubernetes job name
 * @param envVars       environment variables to be provided to the job
 * @param timestamp     timestamp used to create a unique name for the Kubernetes job.
 */
public void runJob(String cronJobName, String jobNamePrefix, List<EnvVar> envVars, String timestamp) {
    try (final KubernetesClient kubernetesClient = new DefaultKubernetesClient()) {
        String jobName = jobNamePrefix + '-' + timestamp;
        final Job job = retrieveOrCreateJob(jobName, cronJobName, envVars, kubernetesClient);
        final CountDownLatch watchLatch = new CountDownLatch(1);
        MardukPodWatcher mardukPodWatcher = new MardukPodWatcher(job, watchLatch, jobName);
        try (Watch watch = kubernetesClient.pods().inNamespace(kubernetesNamespace).withLabel("job-name", jobName).watch(mardukPodWatcher)) {
            boolean jobCompletedBeforeTimeout = watchLatch.await(jobTimeoutSecond, TimeUnit.SECONDS);
            if (!jobCompletedBeforeTimeout) {
                throw new KubernetesJobRunnerException("Timeout while waiting for the Graph Builder job " + jobName + " to complete.");
            }
            JobStatus status = kubernetesClient.batch().v1().jobs().inNamespace(kubernetesNamespace).withName(jobName).get().getStatus();
            LOGGER.debug("Kubernetes Job status on completion: {}", status);
            // test the pod status rather than the job status since the job status may be out of sync with the pod status
            if (mardukPodWatcher.isSucceeded()) {
                LOGGER.info("The Graph Builder job {} completed successfully.", jobName);
            } else if (mardukPodWatcher.isKubernetesClientError()) {
                throw new KubernetesJobRunnerException("Kubernetes client error while watching the Graph Builder job " + jobName);
            } else {
                throw new KubernetesJobRunnerException("The Graph Builder job " + jobName + " failed.");
            }
        } catch (KubernetesClientException e) {
            throw new KubernetesJobRunnerException("Could not watch pod", e);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new KubernetesJobRunnerException("Interrupted while watching pod", e);
        } finally {
            // Delete job after completion unless there was a Kubernetes error that can be retried
            if (!mardukPodWatcher.isKubernetesClientError() && deleteJobAfterCompletion) {
                LOGGER.info("Deleting job {} after completion.", jobName);
                deleteKubernetesJob(kubernetesClient, job);
                LOGGER.info("Deleted job {} after completion.", jobName);
            }
        }
    }
}

Example 28 with Job

use of com.google.cloud.dataproc.v1.Job in project jkube by eclipse.

the class KubernetesHelper method extractPodLabelSelector.

public static LabelSelector extractPodLabelSelector(HasMetadata entity) {
    LabelSelector selector = null;
    if (entity instanceof Deployment) {
        Deployment resource = (Deployment) entity;
        DeploymentSpec spec = resource.getSpec();
        if (spec != null) {
            selector = spec.getSelector();
        }
    } else if (entity instanceof ReplicaSet) {
        ReplicaSet resource = (ReplicaSet) entity;
        ReplicaSetSpec spec = resource.getSpec();
        if (spec != null) {
            selector = spec.getSelector();
        }
    } else if (entity instanceof DeploymentConfig) {
        DeploymentConfig resource = (DeploymentConfig) entity;
        DeploymentConfigSpec spec = resource.getSpec();
        if (spec != null) {
            selector = toLabelSelector(spec.getSelector());
        }
    } else if (entity instanceof ReplicationController) {
        ReplicationController resource = (ReplicationController) entity;
        ReplicationControllerSpec spec = resource.getSpec();
        if (spec != null) {
            selector = toLabelSelector(spec.getSelector());
        }
    } else if (entity instanceof DaemonSet) {
        DaemonSet resource = (DaemonSet) entity;
        DaemonSetSpec spec = resource.getSpec();
        if (spec != null) {
            selector = spec.getSelector();
        }
    } else if (entity instanceof StatefulSet) {
        StatefulSet resource = (StatefulSet) entity;
        StatefulSetSpec spec = resource.getSpec();
        if (spec != null) {
            selector = spec.getSelector();
        }
    } else if (entity instanceof Job) {
        selector = toLabelSelector((Job) entity);
    }
    return selector;
}

Example 29 with Job

use of com.google.cloud.dataproc.v1.Job in project jkube by eclipse.

the class TriggersAnnotationEnricherTest method testNoEnrichment.

@Test
public void testNoEnrichment() {
    KubernetesListBuilder builder = new KubernetesListBuilder().addToItems(new JobBuilder().withNewMetadata().addToAnnotations("dummy", "annotation").endMetadata().withNewSpec().withNewTemplate().withNewSpec().withContainers(createContainers("c1", "is1:latest", "c2", "is2:latest")).endSpec().endTemplate().endSpec().build());
    TriggersAnnotationEnricher enricher = new TriggersAnnotationEnricher(context);
    enricher.enrich(PlatformMode.kubernetes, builder);
    Job res = (Job) builder.build().getItems().get(0);
    String triggers = res.getMetadata().getAnnotations().get("image.openshift.io/triggers");
    assertNull(triggers);
}

Example 30 with Job

use of com.google.cloud.dataproc.v1.Job in project jkube by eclipse.

the class JobHandlerTest method overrideReplicas.

@Test
public void overrideReplicas() {
    // Given
    final KubernetesListBuilder klb = new KubernetesListBuilder().addToItems(new Job());
    // When
    jobHandler.overrideReplicas(klb, 1337);
    // Then
    assertThat(klb.buildItems()).hasSize(1).first().isEqualTo(new Job());
}