Examples with PodInfo io.cdap.cdap.master.environment.k8s.PodInfo used on opensource projects

Example 1 with PodInfo

use of io.cdap.cdap.master.environment.k8s.PodInfo in project cdap by caskdata.

the class KubeTwillPreparer method createPodSpec.

/**
 * Creates a {@link V1PodSpec} for specifying pod information for running the given runnable.
 *
 * @param runtimeConfigLocation the {@link Location} containing the runtime config archive
 * @param runtimeSpecs the specification for the {@link TwillRunnable} and its resources requirements
 * @param restartPolicy pod restart policy
 * @param extraMounts volumes to be mounted
 * @return a {@link V1PodSpec}
 */
private V1PodSpec createPodSpec(Location runtimeConfigLocation, Map<String, RuntimeSpecification> runtimeSpecs, String restartPolicy, List<String> args, V1VolumeMount... extraMounts) {
    String workDir = "/workDir-" + twillRunId.getId();
    V1Volume podInfoVolume = createPodInfoVolume(podInfo);
    RuntimeSpecification mainRuntimeSpec = getMainRuntimeSpecification(runtimeSpecs);
    String runnableName = mainRuntimeSpec.getName();
    V1ResourceRequirements initContainerResourceRequirements = createResourceRequirements(mainRuntimeSpec.getResourceSpecification());
    // Add volume mounts to the container. Add those from the current pod for mount cdap and hadoop conf.
    List<V1VolumeMount> volumeMounts = new ArrayList<>(podInfo.getContainerVolumeMounts());
    volumeMounts.add(new V1VolumeMount().name(podInfoVolume.getName()).mountPath(podInfo.getPodInfoDir()).readOnly(true));
    // Add the working directory the file localization by the init container
    volumeMounts.add(new V1VolumeMount().name("workdir").mountPath(workDir));
    volumeMounts.addAll(Arrays.asList(extraMounts));
    // Mount all volumes including cdap-secret for init container as it runs system/trusted code.
    List<V1VolumeMount> initContainerVolumeMounts = new ArrayList<>(volumeMounts);
    // Mount all except cdap-secret for main container by default. If requested, cdap-secret will
    // get mounted later.
    List<V1VolumeMount> containerVolumeMounts = volumeMounts.stream().filter(v -> !v.getName().equals(KubeMasterEnvironment.SECURITY_CONFIG_NAME)).collect(Collectors.toList());
    // Setup the container environment. Inherit everything from the current pod.
    Map<String, String> initContainerEnvirons = podInfo.getContainerEnvironments().stream().collect(Collectors.toMap(V1EnvVar::getName, V1EnvVar::getValue));
    // Add all environments of the the main runnable for the init container.
    if (environments.get(mainRuntimeSpec.getName()) != null) {
        initContainerEnvirons.putAll(environments.get(mainRuntimeSpec.getName()));
    }
    V1PodSpecBuilder podSpecBuilder = new V1PodSpecBuilder();
    if (schedulerQueue != null) {
        podSpecBuilder = podSpecBuilder.withPriorityClassName(schedulerQueue);
    }
    if (serviceAccountName == null) {
        serviceAccountName = podInfo.getServiceAccountName();
    }
    return podSpecBuilder.withServiceAccountName(serviceAccountName).withRuntimeClassName(podInfo.getRuntimeClassName()).addAllToVolumes(podInfo.getVolumes()).addToVolumes(podInfoVolume, new V1Volume().name("workdir").emptyDir(new V1EmptyDirVolumeSource())).withInitContainers(createContainer("file-localizer", podInfo.getContainerImage(), podInfo.getImagePullPolicy(), workDir, initContainerResourceRequirements, initContainerVolumeMounts, initContainerEnvirons, FileLocalizer.class, runtimeConfigLocation.toURI().toString(), mainRuntimeSpec.getName())).withContainers(createContainers(runtimeSpecs, workDir, containerVolumeMounts, args)).withSecurityContext(podInfo.getSecurityContext()).withRestartPolicy(restartPolicy).build();
}

Example 2 with PodInfo

use of io.cdap.cdap.master.environment.k8s.PodInfo in project cdap by caskdata.

the class KubeTwillLauncher method run.

@Override
public void run(String[] args) throws Exception {
    if (args.length < 1) {
        throw new IllegalArgumentException("Requires runnable name in the argument");
    }
    String runnableName = args[0];
    Path runtimeConfigDir = Paths.get(Constants.Files.RUNTIME_CONFIG_JAR);
    Path argumentsPath = runtimeConfigDir.resolve(Constants.Files.ARGUMENTS);
    // Deserialize the arguments
    List<String> appArgs;
    List<String> runnableArgs;
    try (Reader reader = Files.newBufferedReader(argumentsPath, StandardCharsets.UTF_8)) {
        JsonObject jsonObj = GSON.fromJson(reader, JsonObject.class);
        appArgs = GSON.fromJson(jsonObj.get("arguments"), new TypeToken<List<String>>() {
        }.getType());
        Map<String, List<String>> map = GSON.fromJson(jsonObj.get("runnableArguments"), new TypeToken<Map<String, List<String>>>() {
        }.getType());
        runnableArgs = map.getOrDefault(runnableName, Collections.emptyList());
    }
    PodInfo podInfo = masterEnv.getPodInfo();
    try {
        TwillRuntimeSpecification twillRuntimeSpec = TwillRuntimeSpecificationAdapter.create().fromJson(runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC).toFile());
        RuntimeSpecification runtimeSpec = twillRuntimeSpec.getTwillSpecification().getRunnables().get(runnableName);
        RunId runId = twillRuntimeSpec.getTwillAppRunId();
        String runnableClassName = runtimeSpec.getRunnableSpecification().getClassName();
        Class<?> runnableClass = context.getClass().getClassLoader().loadClass(runnableClassName);
        if (!TwillRunnable.class.isAssignableFrom(runnableClass)) {
            throw new IllegalArgumentException("Class " + runnableClass + " is not an instance of " + TwillRunnable.class);
        }
        twillRunnable = (TwillRunnable) Instances.newInstance(runnableClass);
        try (KubeTwillContext twillContext = new KubeTwillContext(runtimeSpec, runId, RunIds.fromString(runId.getId() + "-0"), appArgs.toArray(new String[0]), runnableArgs.toArray(new String[0]), masterEnv)) {
            twillRunnable.initialize(twillContext);
            if (!stopped) {
                twillRunnable.run();
            }
        }
    } finally {
        try {
            TwillRunnable runnable = twillRunnable;
            if (runnable != null) {
                runnable.destroy();
            }
        } finally {
            if (Arrays.stream(args).noneMatch(str -> str.equalsIgnoreCase(KubeMasterEnvironment.DISABLE_POD_DELETION))) {
                // Delete the pod itself to avoid pod goes into CrashLoopBackoff. This is added for preview pods.
                // When pod is exited, exponential backoff happens. So pod restart time keep increasing.
                // Deleting pod does not trigger exponential backoff.
                // See https://github.com/kubernetes/kubernetes/issues/57291
                deletePod(podInfo);
            }
        }
    }
}