Examples with EfsMount com.netflix.titus.api.model.EfsMount used on opensource projects

Example 1 with EfsMount

use of com.netflix.titus.api.model.EfsMount in project titus-control-plane by Netflix.

the class V1SpecPodFactory method appendEfsMounts.

void appendEfsMounts(V1PodSpec spec, V1Container container, Job<?> job) {
    List<EfsMount> efsMounts = job.getJobDescriptor().getContainer().getContainerResources().getEfsMounts();
    if (efsMounts.isEmpty()) {
        return;
    }
    for (EfsMount efsMount : efsMounts) {
        boolean readOnly = efsMount.getMountPerm() == EfsMount.MountPerm.RO;
        String efsId = efsMount.getEfsId();
        String efsMountPoint = efsMount.getMountPoint();
        String efsRelativeMountPoint = StringExt.isEmpty(efsMount.getEfsRelativeMountPoint()) ? "/" : efsMount.getEfsRelativeMountPoint();
        String name = sanitizeVolumeName(efsId + efsRelativeMountPoint);
        V1VolumeMount volumeMount = new V1VolumeMount().name(name).mountPath(efsMountPoint).readOnly(readOnly);
        container.addVolumeMountsItem(volumeMount);
        // We can't have duplicate volumes in here. In theory there should be a many:one mapping between
        // EFS mounts and the volumes that back them. For example, there could be two mounts to the same
        // nfs server, but with different *mount points*, but there should only be 1 volumes behind them
        List<String> allNames = KubePodUtil.getVolumeNames(spec.getVolumes());
        if (!allNames.contains(name)) {
            V1NFSVolumeSource nfsVolumeSource = new V1NFSVolumeSource().server(efsIdToNFSServer(efsId)).readOnly(false);
            // "path" here represents the server-side relative mount path, sometimes called
            // the "exported directory", and goes into the v1 Volume
            nfsVolumeSource.setPath(efsRelativeMountPoint);
            V1Volume volume = new V1Volume().name(name).nfs(nfsVolumeSource);
            spec.addVolumesItem(volume);
        }
    }
}

Example 2 with EfsMount

use of com.netflix.titus.api.model.EfsMount in project titus-control-plane by Netflix.

the class V1SpecPodFactoryTest method testEFSMountsHandlesDuplicateVolumes.

@Test
public void testEFSMountsHandlesDuplicateVolumes() {
    Job<BatchJobExt> job = JobGenerator.oneBatchJob();
    BatchJobTask task = JobGenerator.oneBatchTask();
    EfsMount newEfsMount = new EfsMount("1.2.3.4", "/mountpoint", EfsMount.MountPerm.RO, "/relative");
    EfsMount newEfsMount2 = new EfsMount("1.2.3.4", "/mountpoint2", EfsMount.MountPerm.RO, "/relative");
    EfsMount newEfsMount3 = new EfsMount("1.2.3.4", "/mountpoint3", EfsMount.MountPerm.RW, "/relative");
    Container newContainer = job.getJobDescriptor().getContainer();
    ContainerResources newContainerResources = newContainer.getContainerResources();
    Container newContainerWithEFS = newContainer.toBuilder().withContainerResources(newContainerResources.newBuilder().withEfsMounts(Arrays.asList(newEfsMount, newEfsMount2, newEfsMount3)).build()).build();
    job = job.toBuilder().withJobDescriptor(job.getJobDescriptor().toBuilder().withContainer(newContainerWithEFS).build()).build();
    when(podAffinityFactory.buildV1Affinity(job, task)).thenReturn(Pair.of(new V1Affinity(), new HashMap<>()));
    V1Pod pod = podFactory.buildV1Pod(job, task);
    // Part 1: There should only be *one* EFS volume to share
    List<V1Volume> volumes = pod.getSpec().getVolumes();
    // one for nfs, one for shm
    assertThat(volumes.size()).isEqualTo(2);
    V1Volume v1NFSVolume = volumes.get(0);
    assertThat(v1NFSVolume.getName()).isEqualTo("1-2-3-4-relative-vol");
    assertThat(v1NFSVolume.getNfs().getServer()).isEqualTo("1.2.3.4");
    assertThat(v1NFSVolume.getNfs().getPath()).isEqualTo("/relative");
    // All NFS volumes that are generated like this should be RW, and
    // delegating the actual RO/RW state to the volume *mount*.
    assertThat(v1NFSVolume.getNfs().getReadOnly()).isEqualTo(false);
    // Part 2: there should be *3* volume mounts, all sharing the volume
    List<V1VolumeMount> vms = pod.getSpec().getContainers().get(0).getVolumeMounts();
    // 3 for nfs, one for shm
    assertThat(vms.size()).isEqualTo(4);
    V1VolumeMount v1NFSvm1 = vms.get(0);
    assertThat(v1NFSvm1.getName()).isEqualTo("1-2-3-4-relative-vol");
    assertThat(v1NFSvm1.getMountPath()).isEqualTo("/mountpoint");
    assertThat(v1NFSvm1.getReadOnly()).isTrue();
    V1VolumeMount v1NFSvm2 = vms.get(1);
    assertThat(v1NFSvm2.getName()).isEqualTo("1-2-3-4-relative-vol");
    assertThat(v1NFSvm2.getMountPath()).isEqualTo("/mountpoint2");
    assertThat(v1NFSvm2.getReadOnly()).isTrue();
    V1VolumeMount v1NFSvm3 = vms.get(2);
    assertThat(v1NFSvm3.getName()).isEqualTo("1-2-3-4-relative-vol");
    assertThat(v1NFSvm3.getMountPath()).isEqualTo("/mountpoint3");
    assertThat(v1NFSvm3.getReadOnly()).isFalse();
}

Example 3 with EfsMount

use of com.netflix.titus.api.model.EfsMount in project titus-control-plane by Netflix.

the class V1SpecPodFactoryTest method testEFSMountsGetTransformedSafely.

@Test
public void testEFSMountsGetTransformedSafely() {
    Job<BatchJobExt> job = JobGenerator.oneBatchJob();
    BatchJobTask task = JobGenerator.oneBatchTask();
    EfsMount newEfsMount = new EfsMount("1.2.3.4", "/mountpoint", EfsMount.MountPerm.RO, "/relative/");
    Container newContainer = job.getJobDescriptor().getContainer();
    ContainerResources newContainerResources = newContainer.getContainerResources();
    Container newContainerWithEFS = newContainer.toBuilder().withContainerResources(newContainerResources.newBuilder().withEfsMounts(Collections.singletonList(newEfsMount)).build()).build();
    job = job.toBuilder().withJobDescriptor(job.getJobDescriptor().toBuilder().withContainer(newContainerWithEFS).build()).build();
    when(podAffinityFactory.buildV1Affinity(job, task)).thenReturn(Pair.of(new V1Affinity(), new HashMap<>()));
    V1Pod pod = podFactory.buildV1Pod(job, task);
    // Part 1: the volume section needs to be well-formed
    List<V1Volume> volumes = pod.getSpec().getVolumes();
    // one for nfs, one for shm
    assertThat(volumes.size()).isEqualTo(2);
    V1Volume v1NFSVolume = volumes.get(0);
    assertThat(v1NFSVolume.getName()).isEqualTo("1-2-3-4-relative--vol");
    assertThat(v1NFSVolume.getNfs().getServer()).isEqualTo("1.2.3.4");
    assertThat(v1NFSVolume.getNfs().getPath()).isEqualTo("/relative/");
    assertThat(v1NFSVolume.getNfs().getReadOnly()).isEqualTo(false);
    // Part 2: the volume mount section needs to applied to the first container in the podspec
    List<V1VolumeMount> vms = pod.getSpec().getContainers().get(0).getVolumeMounts();
    // one for nfs, one for shm
    assertThat(vms.size()).isEqualTo(2);
    V1VolumeMount v1NFSvm = vms.get(0);
    assertThat(v1NFSvm.getName()).isEqualTo("1-2-3-4-relative--vol");
    assertThat(v1NFSvm.getMountPath()).isEqualTo("/mountpoint");
    assertThat(v1NFSvm.getReadOnly()).isEqualTo(true);
}

Example 4 with EfsMount

use of com.netflix.titus.api.model.EfsMount in project titus-control-plane by Netflix.

the class JobModelSanitizationTest method testBatchJobWithIncompleteEfsDefinition.

@Test
public void testBatchJobWithIncompleteEfsDefinition() {
    JobDescriptor<BatchJobExt> jobDescriptor = oneTaskBatchJobDescriptor();
    JobDescriptor<BatchJobExt> incompleteEfsDefinition = JobModel.newJobDescriptor(jobDescriptor).withContainer(JobModel.newContainer(jobDescriptor.getContainer()).withContainerResources(JobModel.newContainerResources(jobDescriptor.getContainer().getContainerResources()).withEfsMounts(Collections.singletonList(new EfsMount("efsId#1", "/data", null, null))).build()).build()).build();
    Job<BatchJobExt> job = JobGenerator.batchJobs(incompleteEfsDefinition).getValue();
    // EFS violation expected
    assertThat(entitySanitizer.validate(job)).hasSize(1);
    // Now do cleanup
    Job<BatchJobExt> sanitized = entitySanitizer.sanitize(job).get();
    assertThat(entitySanitizer.validate(sanitized)).isEmpty();
}

Example 5 with EfsMount

use of com.netflix.titus.api.model.EfsMount in project titus-control-plane by Netflix.

the class JobSubmitAndControlBasicTest method testSubmitBatchJobWithEfsMount.

/**
 * Verify batch job submit with the expected state transitions. Verify agent receives proper EFS mount data.
 */
@Test(timeout = 30_000)
public void testSubmitBatchJobWithEfsMount() {
    EfsMount efsMount1 = ContainersGenerator.efsMounts().getValue().toBuilder().withMountPoint("/data/logs").build();
    EfsMount efsMount2 = ContainersGenerator.efsMounts().skip(1).getValue().toBuilder().withMountPoint("/data").build();
    List<EfsMount> efsMounts = asList(efsMount1, efsMount2);
    List<EfsMount> expectedOrder = asList(efsMount2, efsMount1);
    JobDescriptor<BatchJobExt> jobWithEfs = ONE_TASK_BATCH_JOB.but(jd -> jd.getContainer().but(c -> c.getContainerResources().toBuilder().withEfsMounts(efsMounts)));
    jobsScenarioBuilder.schedule(jobWithEfs, jobScenarioBuilder -> jobScenarioBuilder.template(ScenarioTemplates.startTasksInNewJob()).assertEachPod(podWithEfsMounts(expectedOrder), "Container not assigned the expected EFS mount").allTasks(ScenarioTemplates.completeTask()).template(ScenarioTemplates.jobFinished()).expectJobEventStreamCompletes());
}