Examples with V1Container io.kubernetes.client.openapi.models.V1Container used on opensource projects

Example 6 with V1Container

use of io.kubernetes.client.openapi.models.V1Container in project twister2 by DSC-SPIDAL.

the class RequestObjectBuilder method constructContainer.

/**
 * construct a container
 */
public static V1Container constructContainer(ComputeResource computeResource, int containerIndex) {
    // construct container and add it to podSpec
    V1Container container = new V1Container();
    String containerName = KubernetesUtils.createContainerName(containerIndex);
    container.setName(containerName);
    String containerImage = KubernetesContext.twister2DockerImageForK8s(config);
    if (containerImage == null) {
        throw new RuntimeException("Container Image name is null. Config parameter: " + "twister2.resource.kubernetes.docker.image can not be null");
    }
    container.setImage(containerImage);
    container.setImagePullPolicy(KubernetesContext.imagePullPolicy(config));
    String startScript = null;
    double cpuPerContainer = computeResource.getCpu();
    int ramPerContainer = computeResource.getRamMegaBytes();
    if (SchedulerContext.usingOpenMPI(config)) {
        startScript = "./init_openmpi.sh";
        cpuPerContainer = cpuPerContainer * computeResource.getWorkersPerPod();
        ramPerContainer = ramPerContainer * computeResource.getWorkersPerPod();
    } else {
        startScript = "./init.sh";
    }
    container.setCommand(Arrays.asList("/bin/bash"));
    container.setArgs(Arrays.asList("-c", startScript));
    V1ResourceRequirements resReq = new V1ResourceRequirements();
    if (KubernetesContext.bindWorkerToCPU(config)) {
        resReq.putLimitsItem("cpu", new Quantity(String.format("%.2f", cpuPerContainer)));
        resReq.putLimitsItem("memory", new Quantity(ramPerContainer + "Mi"));
    } else {
        resReq.putRequestsItem("cpu", new Quantity(String.format("%.2f", cpuPerContainer)));
        resReq.putRequestsItem("memory", new Quantity(ramPerContainer + "Mi"));
    }
    container.setResources(resReq);
    ArrayList<V1VolumeMount> volumeMounts = new ArrayList<>();
    V1VolumeMount memoryVolumeMount = new V1VolumeMount();
    memoryVolumeMount.setName(KubernetesConstants.POD_MEMORY_VOLUME_NAME);
    memoryVolumeMount.setMountPath(KubernetesConstants.POD_MEMORY_VOLUME);
    volumeMounts.add(memoryVolumeMount);
    if (computeResource.getDiskGigaBytes() > 0) {
        V1VolumeMount volatileVolumeMount = new V1VolumeMount();
        volatileVolumeMount.setName(KubernetesConstants.POD_VOLATILE_VOLUME_NAME);
        volatileVolumeMount.setMountPath(KubernetesConstants.POD_VOLATILE_VOLUME);
        volumeMounts.add(volatileVolumeMount);
    }
    if (SchedulerContext.persistentVolumeRequested(config)) {
        V1VolumeMount persVolumeMount = new V1VolumeMount();
        persVolumeMount.setName(KubernetesConstants.PERSISTENT_VOLUME_NAME);
        persVolumeMount.setMountPath(KubernetesConstants.PERSISTENT_VOLUME_MOUNT);
        volumeMounts.add(persVolumeMount);
    }
    // mount Secret object as a volume
    if (SchedulerContext.usingOpenMPI(config)) {
        V1VolumeMount persVolumeMount = new V1VolumeMount();
        persVolumeMount.setName(KubernetesConstants.SECRET_VOLUME_NAME);
        persVolumeMount.setMountPath(KubernetesConstants.SECRET_VOLUME_MOUNT);
        volumeMounts.add(persVolumeMount);
    }
    container.setVolumeMounts(volumeMounts);
    int containerPort = KubernetesContext.workerBasePort(config) + containerIndex * (SchedulerContext.numberOfAdditionalPorts(config) + 1);
    V1ContainerPort port = new V1ContainerPort();
    // currently not used
    port.name("port11");
    port.containerPort(containerPort);
    port.setProtocol(KubernetesContext.workerTransportProtocol(config));
    container.setPorts(Arrays.asList(port));
    int jvmMemory = (int) (computeResource.getRamMegaBytes() * KubernetesContext.jvmMemoryFraction(config));
    container.setEnv(constructEnvironmentVariables(containerName, containerPort, jvmMemory));
    return container;
}

Example 7 with V1Container

use of io.kubernetes.client.openapi.models.V1Container in project twister2 by DSC-SPIDAL.

the class JobLogger method watchPodsToRunningStartLoggers.

/**
 * watch job pods until they become Running and start loggers for each container afterward
 */
private void watchPodsToRunningStartLoggers() {
    String jobPodsLabel = KubernetesUtils.jobLabelSelector(job.getJobId());
    Integer timeoutSeconds = Integer.MAX_VALUE;
    try {
        watcher = Watch.createWatch(KubernetesController.getApiClient(), v1Api.listNamespacedPodCall(namespace, null, null, null, null, jobPodsLabel, null, null, timeoutSeconds, Boolean.TRUE, null), new TypeToken<Watch.Response<V1Pod>>() {
        }.getType());
    } catch (ApiException e) {
        String logMessage = "Exception when watching the pods to get the IPs: \n" + "exCode: " + e.getCode() + "\n" + "responseBody: " + e.getResponseBody();
        LOG.log(Level.SEVERE, logMessage, e);
        throw new RuntimeException(e);
    }
    // we catch this exception and ignore it.
    try {
        for (Watch.Response<V1Pod> item : watcher) {
            if (stopLogger) {
                break;
            }
            // it means that the pod is in the process of being deleted
            if (item.object != null && item.object.getMetadata().getName().startsWith(job.getJobId()) && KubernetesUtils.isPodRunning(item.object)) {
                String podName = item.object.getMetadata().getName();
                String podIP = item.object.getStatus().getPodIP();
                List<V1Container> containers = item.object.getSpec().getContainers();
                if (podName.endsWith("-jm-0")) {
                    String contName = containers.get(0).getName();
                    String id = "job-master-ip" + podIP;
                    WorkerLogger workerLogger = new WorkerLogger(namespace, podName, contName, id, logsDir, v1Api, this);
                    startWorkerLogger(workerLogger);
                    continue;
                }
                for (V1Container container : containers) {
                    int wID = K8sWorkerUtils.calculateWorkerID(job, podName, container.getName());
                    // this means job is scaled up
                    if (wID >= numberOfWorkers) {
                        numberOfWorkers = wID + 1;
                    }
                    String id = "worker" + wID + "-ip" + podIP;
                    WorkerLogger workerLogger = new WorkerLogger(namespace, podName, container.getName(), id, logsDir, v1Api, this);
                    startWorkerLogger(workerLogger);
                }
            }
        }
    } catch (RuntimeException e) {
        if (stopLogger) {
            LOG.fine("JobLogger is stopped.");
            return;
        } else {
            throw e;
        }
    }
    closeWatcher();
}

Example 8 with V1Container

use of io.kubernetes.client.openapi.models.V1Container in project heron by twitter.

the class V1Controller method mountSecretsAsVolumes.

/**
 * Adds <code>Volume Mounts</code> for <code>Secrets</code> to a pod.
 * @param podSpec <code>Pod Spec</code> to add secrets to.
 */
private void mountSecretsAsVolumes(V1PodSpec podSpec) {
    final Config config = getConfiguration();
    final Map<String, String> secrets = KubernetesContext.getPodSecretsToMount(config);
    for (Map.Entry<String, String> secret : secrets.entrySet()) {
        final V1VolumeMount mount = new V1VolumeMount().name(secret.getKey()).mountPath(secret.getValue());
        final V1Volume secretVolume = new V1Volume().name(secret.getKey()).secret(new V1SecretVolumeSourceBuilder().withSecretName(secret.getKey()).build());
        podSpec.addVolumesItem(secretVolume);
        for (V1Container container : podSpec.getContainers()) {
            container.addVolumeMountsItem(mount);
        }
    }
}

Example 9 with V1Container

use of io.kubernetes.client.openapi.models.V1Container in project heron by twitter.

the class V1Controller method configurePodSpec.

/**
 * Configures the <code>Pod Spec</code> section of the <code>StatefulSet</code>. The <code>Heron</code> container
 * will be configured to allow it to function but other supplied containers are loaded verbatim.
 * @param podTemplateSpec The <code>Pod Template Spec</code> section to update.
 * @param resource Passed down to configure the resource limits.
 * @param numberOfInstances Passed down to configure the ports.
 * @param isExecutor Flag used to configure components specific to <code>Executor</code> and <code>Manager</code>.
 * @param volumes <code>Volumes</code> generated from configurations options.
 * @param volumeMounts <code>Volume Mounts</code> generated from configurations options.
 */
private void configurePodSpec(final V1PodTemplateSpec podTemplateSpec, Resource resource, int numberOfInstances, boolean isExecutor, List<V1Volume> volumes, List<V1VolumeMount> volumeMounts) {
    if (podTemplateSpec.getSpec() == null) {
        podTemplateSpec.setSpec(new V1PodSpec());
    }
    final V1PodSpec podSpec = podTemplateSpec.getSpec();
    // Set the termination period to 0 so pods can be deleted quickly
    podSpec.setTerminationGracePeriodSeconds(0L);
    // Set the pod tolerations so pods are rescheduled when nodes go down
    // https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions
    configureTolerations(podSpec);
    // Get <Heron> container and ignore all others.
    final String containerName = isExecutor ? KubernetesConstants.EXECUTOR_NAME : KubernetesConstants.MANAGER_NAME;
    V1Container heronContainer = null;
    List<V1Container> containers = podSpec.getContainers();
    if (containers != null) {
        for (V1Container container : containers) {
            final String name = container.getName();
            if (name != null && name.equals(containerName)) {
                if (heronContainer != null) {
                    throw new TopologySubmissionException(String.format("Multiple configurations found for '%s' container", containerName));
                }
                heronContainer = container;
            }
        }
    } else {
        containers = new LinkedList<>();
    }
    if (heronContainer == null) {
        heronContainer = new V1Container().name(containerName);
        containers.add(heronContainer);
    }
    if (!volumes.isEmpty() || !volumeMounts.isEmpty()) {
        configurePodWithVolumesAndMountsFromCLI(podSpec, heronContainer, volumes, volumeMounts);
    }
    configureHeronContainer(resource, numberOfInstances, heronContainer, isExecutor);
    podSpec.setContainers(containers);
    addVolumesIfPresent(podSpec);
    mountSecretsAsVolumes(podSpec);
}

Example 10 with V1Container

use of io.kubernetes.client.openapi.models.V1Container in project heron by twitter.

the class V1ControllerTest method testConfigureContainerEnvVars.

@Test
public void testConfigureContainerEnvVars() {
    final List<V1EnvVar> heronEnvVars = Collections.unmodifiableList(V1Controller.getExecutorEnvVars());
    final V1EnvVar additionEnvVar = new V1EnvVar().name("env-variable-to-be-kept").valueFrom(new V1EnvVarSource().fieldRef(new V1ObjectFieldSelector().fieldPath("env-variable-was-kept")));
    final List<V1EnvVar> inputEnvVars = Arrays.asList(new V1EnvVar().name(KubernetesConstants.ENV_HOST).valueFrom(new V1EnvVarSource().fieldRef(new V1ObjectFieldSelector().fieldPath("env-host-to-be-replaced"))), new V1EnvVar().name(KubernetesConstants.ENV_POD_NAME).valueFrom(new V1EnvVarSource().fieldRef(new V1ObjectFieldSelector().fieldPath("pod-name-to-be-replaced"))), additionEnvVar);
    // Null env vars. This is the default case.
    V1Container containerWithNullEnvVars = new V1ContainerBuilder().build();
    v1ControllerWithPodTemplate.configureContainerEnvVars(containerWithNullEnvVars);
    Assert.assertTrue("ENV_HOST & ENV_POD_NAME in container with null Env Vars should match", CollectionUtils.containsAll(containerWithNullEnvVars.getEnv(), heronEnvVars));
    // Empty env vars.
    V1Container containerWithEmptyEnvVars = new V1ContainerBuilder().withEnv(new LinkedList<>()).build();
    v1ControllerWithPodTemplate.configureContainerEnvVars(containerWithEmptyEnvVars);
    Assert.assertTrue("ENV_HOST & ENV_POD_NAME in container with empty Env Vars should match", CollectionUtils.containsAll(containerWithEmptyEnvVars.getEnv(), heronEnvVars));
    // Env Var overriding.
    final List<V1EnvVar> expectedOverriding = new LinkedList<>(heronEnvVars);
    expectedOverriding.add(additionEnvVar);
    V1Container containerWithEnvVars = new V1ContainerBuilder().withEnv(inputEnvVars).build();
    v1ControllerWithPodTemplate.configureContainerEnvVars(containerWithEnvVars);
    Assert.assertTrue("ENV_HOST & ENV_POD_NAME in container with Env Vars should be overridden", CollectionUtils.containsAll(containerWithEnvVars.getEnv(), expectedOverriding));
}