Search in sources :

Example 1 with KeyToPath

use of io.fabric8.kubernetes.api.model.KeyToPath in project flink by apache.

the class HadoopConfMountDecorator method decorateFlinkPod.

@Override
public FlinkPod decorateFlinkPod(FlinkPod flinkPod) {
    Volume hadoopConfVolume;
    final Optional<String> existingConfigMap = kubernetesParameters.getExistingHadoopConfigurationConfigMap();
    if (existingConfigMap.isPresent()) {
        hadoopConfVolume = new VolumeBuilder().withName(Constants.HADOOP_CONF_VOLUME).withNewConfigMap().withName(existingConfigMap.get()).endConfigMap().build();
    } else {
        final Optional<String> localHadoopConfigurationDirectory = kubernetesParameters.getLocalHadoopConfigurationDirectory();
        if (!localHadoopConfigurationDirectory.isPresent()) {
            return flinkPod;
        }
        final List<File> hadoopConfigurationFileItems = getHadoopConfigurationFileItems(localHadoopConfigurationDirectory.get());
        if (hadoopConfigurationFileItems.isEmpty()) {
            LOG.warn("Found 0 files in directory {}, skip to mount the Hadoop Configuration ConfigMap.", localHadoopConfigurationDirectory.get());
            return flinkPod;
        }
        final List<KeyToPath> keyToPaths = hadoopConfigurationFileItems.stream().map(file -> new KeyToPathBuilder().withKey(file.getName()).withPath(file.getName()).build()).collect(Collectors.toList());
        hadoopConfVolume = new VolumeBuilder().withName(Constants.HADOOP_CONF_VOLUME).withNewConfigMap().withName(getHadoopConfConfigMapName(kubernetesParameters.getClusterId())).withItems(keyToPaths).endConfigMap().build();
    }
    final Pod podWithHadoopConf = new PodBuilder(flinkPod.getPodWithoutMainContainer()).editOrNewSpec().addNewVolumeLike(hadoopConfVolume).endVolume().endSpec().build();
    final Container containerWithHadoopConf = new ContainerBuilder(flinkPod.getMainContainer()).addNewVolumeMount().withName(Constants.HADOOP_CONF_VOLUME).withMountPath(Constants.HADOOP_CONF_DIR_IN_POD).endVolumeMount().addNewEnv().withName(Constants.ENV_HADOOP_CONF_DIR).withValue(Constants.HADOOP_CONF_DIR_IN_POD).endEnv().build();
    return new FlinkPod.Builder(flinkPod).withPod(podWithHadoopConf).withMainContainer(containerWithHadoopConf).build();
}
Also used : Arrays(java.util.Arrays) Container(io.fabric8.kubernetes.api.model.Container) FileUtils(org.apache.flink.util.FileUtils) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) AbstractKubernetesParameters(org.apache.flink.kubernetes.kubeclient.parameters.AbstractKubernetesParameters) PodBuilder(io.fabric8.kubernetes.api.model.PodBuilder) Map(java.util.Map) KeyToPathBuilder(io.fabric8.kubernetes.api.model.KeyToPathBuilder) ContainerBuilder(io.fabric8.kubernetes.api.model.ContainerBuilder) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) Volume(io.fabric8.kubernetes.api.model.Volume) KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) Logger(org.slf4j.Logger) Pod(io.fabric8.kubernetes.api.model.Pod) IOException(java.io.IOException) VolumeBuilder(io.fabric8.kubernetes.api.model.VolumeBuilder) FlinkPod(org.apache.flink.kubernetes.kubeclient.FlinkPod) HasMetadata(io.fabric8.kubernetes.api.model.HasMetadata) Collectors(java.util.stream.Collectors) File(java.io.File) ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap) ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder) List(java.util.List) Optional(java.util.Optional) Constants(org.apache.flink.kubernetes.utils.Constants) Collections(java.util.Collections) Pod(io.fabric8.kubernetes.api.model.Pod) FlinkPod(org.apache.flink.kubernetes.kubeclient.FlinkPod) FlinkPod(org.apache.flink.kubernetes.kubeclient.FlinkPod) PodBuilder(io.fabric8.kubernetes.api.model.PodBuilder) VolumeBuilder(io.fabric8.kubernetes.api.model.VolumeBuilder) KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) Container(io.fabric8.kubernetes.api.model.Container) ContainerBuilder(io.fabric8.kubernetes.api.model.ContainerBuilder) Volume(io.fabric8.kubernetes.api.model.Volume) KeyToPathBuilder(io.fabric8.kubernetes.api.model.KeyToPathBuilder) File(java.io.File)

Example 2 with KeyToPath

use of io.fabric8.kubernetes.api.model.KeyToPath in project flink by apache.

the class PodTemplateMountDecorator method decoratePod.

private Pod decoratePod(Pod pod) {
    final List<KeyToPath> keyToPaths = new ArrayList<>();
    keyToPaths.add(new KeyToPathBuilder().withKey(TASK_MANAGER_POD_TEMPLATE_FILE_NAME).withPath(TASK_MANAGER_POD_TEMPLATE_FILE_NAME).build());
    final Volume podTemplateVolume = new VolumeBuilder().withName(POD_TEMPLATE_VOLUME).withNewConfigMap().withName(podTemplateConfigMapName).withItems(keyToPaths).endConfigMap().build();
    return new PodBuilder(pod).editSpec().addNewVolumeLike(podTemplateVolume).endVolume().endSpec().build();
}
Also used : KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) Volume(io.fabric8.kubernetes.api.model.Volume) KeyToPathBuilder(io.fabric8.kubernetes.api.model.KeyToPathBuilder) PodBuilder(io.fabric8.kubernetes.api.model.PodBuilder) ArrayList(java.util.ArrayList) VolumeBuilder(io.fabric8.kubernetes.api.model.VolumeBuilder)

Example 3 with KeyToPath

use of io.fabric8.kubernetes.api.model.KeyToPath in project flink by apache.

the class PodTemplateMountDecoratorTest method testDecoratedFlinkPodWithTaskManagerPodTemplate.

@Test
public void testDecoratedFlinkPodWithTaskManagerPodTemplate() throws Exception {
    KubernetesTestUtils.createTemporyFile(POD_TEMPLATE_DATA, flinkConfDir, POD_TEMPLATE_FILE_NAME);
    final FlinkPod resultFlinkPod = podTemplateMountDecorator.decorateFlinkPod(baseFlinkPod);
    final List<KeyToPath> expectedKeyToPaths = Collections.singletonList(new KeyToPathBuilder().withKey(TASK_MANAGER_POD_TEMPLATE_FILE_NAME).withPath(TASK_MANAGER_POD_TEMPLATE_FILE_NAME).build());
    final List<Volume> expectedVolumes = getExpectedVolumes(expectedKeyToPaths);
    assertThat(resultFlinkPod.getPodWithoutMainContainer().getSpec().getVolumes(), containsInAnyOrder(expectedVolumes.toArray()));
    final List<VolumeMount> expectedVolumeMounts = Collections.singletonList(new VolumeMountBuilder().withName(Constants.POD_TEMPLATE_VOLUME).withMountPath(Constants.POD_TEMPLATE_DIR_IN_POD).build());
    assertThat(resultFlinkPod.getMainContainer().getVolumeMounts(), containsInAnyOrder(expectedVolumeMounts.toArray()));
}
Also used : KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) FlinkPod(org.apache.flink.kubernetes.kubeclient.FlinkPod) Volume(io.fabric8.kubernetes.api.model.Volume) KeyToPathBuilder(io.fabric8.kubernetes.api.model.KeyToPathBuilder) VolumeMount(io.fabric8.kubernetes.api.model.VolumeMount) VolumeMountBuilder(io.fabric8.kubernetes.api.model.VolumeMountBuilder) Test(org.junit.Test)

Example 4 with KeyToPath

use of io.fabric8.kubernetes.api.model.KeyToPath in project flink by apache.

the class HadoopConfMountDecoratorTest method testPodWithHadoopConfVolume.

@Test
public void testPodWithHadoopConfVolume() throws IOException {
    setHadoopConfDirEnv();
    generateHadoopConfFileItems();
    final FlinkPod resultFlinkPod = hadoopConfMountDecorator.decorateFlinkPod(baseFlinkPod);
    final List<Volume> resultVolumes = resultFlinkPod.getPodWithoutMainContainer().getSpec().getVolumes();
    assertEquals(1, resultVolumes.size());
    final Volume resultVolume = resultVolumes.get(0);
    assertEquals(Constants.HADOOP_CONF_VOLUME, resultVolume.getName());
    final ConfigMapVolumeSource resultVolumeConfigMap = resultVolume.getConfigMap();
    assertEquals(HadoopConfMountDecorator.getHadoopConfConfigMapName(CLUSTER_ID), resultVolumeConfigMap.getName());
    final Map<String, String> expectedKeyToPaths = new HashMap<String, String>() {

        {
            put("hdfs-site.xml", "hdfs-site.xml");
            put("core-site.xml", "core-site.xml");
        }
    };
    final Map<String, String> resultKeyToPaths = resultVolumeConfigMap.getItems().stream().collect(Collectors.toMap(KeyToPath::getKey, KeyToPath::getPath));
    assertEquals(expectedKeyToPaths, resultKeyToPaths);
}
Also used : FlinkPod(org.apache.flink.kubernetes.kubeclient.FlinkPod) Volume(io.fabric8.kubernetes.api.model.Volume) HashMap(java.util.HashMap) ConfigMapVolumeSource(io.fabric8.kubernetes.api.model.ConfigMapVolumeSource) Test(org.junit.Test)

Example 5 with KeyToPath

use of io.fabric8.kubernetes.api.model.KeyToPath in project flink by apache.

the class FlinkConfMountDecoratorTest method testDecoratedFlinkPodWithoutLog4jAndLogback.

@Test
public void testDecoratedFlinkPodWithoutLog4jAndLogback() {
    final FlinkPod resultFlinkPod = flinkConfMountDecorator.decorateFlinkPod(baseFlinkPod);
    final List<KeyToPath> expectedKeyToPaths = Collections.singletonList(new KeyToPathBuilder().withKey(FLINK_CONF_FILENAME).withPath(FLINK_CONF_FILENAME).build());
    final List<Volume> expectedVolumes = Collections.singletonList(new VolumeBuilder().withName(Constants.FLINK_CONF_VOLUME).withNewConfigMap().withName(getFlinkConfConfigMapName(CLUSTER_ID)).withItems(expectedKeyToPaths).endConfigMap().build());
    assertEquals(expectedVolumes, resultFlinkPod.getPodWithoutMainContainer().getSpec().getVolumes());
    final List<VolumeMount> expectedVolumeMounts = Collections.singletonList(new VolumeMountBuilder().withName(Constants.FLINK_CONF_VOLUME).withMountPath(FLINK_CONF_DIR_IN_POD).build());
    assertEquals(expectedVolumeMounts, resultFlinkPod.getMainContainer().getVolumeMounts());
}
Also used : KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) FlinkPod(org.apache.flink.kubernetes.kubeclient.FlinkPod) Volume(io.fabric8.kubernetes.api.model.Volume) KeyToPathBuilder(io.fabric8.kubernetes.api.model.KeyToPathBuilder) VolumeMount(io.fabric8.kubernetes.api.model.VolumeMount) VolumeBuilder(io.fabric8.kubernetes.api.model.VolumeBuilder) VolumeMountBuilder(io.fabric8.kubernetes.api.model.VolumeMountBuilder) Test(org.junit.Test)

Aggregations

Volume (io.fabric8.kubernetes.api.model.Volume)9 KeyToPath (io.fabric8.kubernetes.api.model.KeyToPath)8 KeyToPathBuilder (io.fabric8.kubernetes.api.model.KeyToPathBuilder)8 FlinkPod (org.apache.flink.kubernetes.kubeclient.FlinkPod)8 VolumeBuilder (io.fabric8.kubernetes.api.model.VolumeBuilder)7 Test (org.junit.Test)6 PodBuilder (io.fabric8.kubernetes.api.model.PodBuilder)3 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 ConfigMap (io.fabric8.kubernetes.api.model.ConfigMap)2 ConfigMapBuilder (io.fabric8.kubernetes.api.model.ConfigMapBuilder)2 Container (io.fabric8.kubernetes.api.model.Container)2 ContainerBuilder (io.fabric8.kubernetes.api.model.ContainerBuilder)2 HasMetadata (io.fabric8.kubernetes.api.model.HasMetadata)2 Pod (io.fabric8.kubernetes.api.model.Pod)2 VolumeMount (io.fabric8.kubernetes.api.model.VolumeMount)2 VolumeMountBuilder (io.fabric8.kubernetes.api.model.VolumeMountBuilder)2 File (java.io.File)2 IOException (java.io.IOException)2 Collections (java.util.Collections)2