Examples with SingularityVolume com.hubspot.mesos.SingularityVolume used on opensource projects

Example 1 with SingularityVolume

use of com.hubspot.mesos.SingularityVolume in project Singularity by HubSpot.

the class SingularitySchedulerTest method dockerDeployWithPorts.

private SingularityDeployBuilder dockerDeployWithPorts() {
    final SingularityDockerPortMapping literalMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>absent(), 80, Optional.of(SingularityPortMappingType.LITERAL), 8080, Optional.<String>absent());
    final SingularityDockerPortMapping offerMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>absent(), 81, Optional.of(SingularityPortMappingType.FROM_OFFER), 0, Optional.of("udp"));
    final SingularityContainerInfo containerInfo = new SingularityContainerInfo(SingularityContainerType.DOCKER, Optional.<List<SingularityVolume>>absent(), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.BRIDGE, Optional.of(Arrays.asList(literalMapping, offerMapping)), Optional.of(false), Optional.of(ImmutableMap.of("env", "var=value")), Optional.absent())));
    final SingularityDeployBuilder deployBuilder = new SingularityDeployBuilder(requestId, "test-docker-ports-deploy");
    deployBuilder.setContainerInfo(Optional.of(containerInfo));
    return deployBuilder;
}

Example 2 with SingularityVolume

use of com.hubspot.mesos.SingularityVolume in project Singularity by HubSpot.

the class SingularityMesosTaskBuilderTest method testDockerTask.

@Test
public void testDockerTask() {
    taskResources = new Resources(1, 1, 1, 0);
    final Protos.Resource portsResource = Protos.Resource.newBuilder().setName("ports").setType(Protos.Value.Type.RANGES).setRanges(Protos.Value.Ranges.newBuilder().addRange(Protos.Value.Range.newBuilder().setBegin(31000).setEnd(31000).build()).build()).build();
    final SingularityDockerPortMapping literalMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>absent(), 80, Optional.of(SingularityPortMappingType.LITERAL), 8080, Optional.<String>absent());
    final SingularityDockerPortMapping offerMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>absent(), 81, Optional.of(SingularityPortMappingType.FROM_OFFER), 0, Optional.of("udp"));
    final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build();
    final SingularityContainerInfo containerInfo = new SingularityContainerInfo(SingularityContainerType.DOCKER, Optional.of(Arrays.asList(new SingularityVolume("/container", Optional.of("/host"), SingularityDockerVolumeMode.RW), new SingularityVolume("/container/${TASK_REQUEST_ID}/${TASK_DEPLOY_ID}", Optional.of("/host/${TASK_ID}"), SingularityDockerVolumeMode.RO))), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.BRIDGE, Optional.of(Arrays.asList(literalMapping, offerMapping)), Optional.of(false), Optional.<Map<String, String>>of(ImmutableMap.of("env", "var=value")), Optional.absent())));
    final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1").setContainerInfo(Optional.of(containerInfo)).setCommand(Optional.of("/bin/echo")).setArguments(Optional.of(Collections.singletonList("wat"))).build();
    final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources);
    assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue());
    assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount());
    assertEquals("wat", task.getMesosTask().getCommand().getArguments(0));
    assertFalse(task.getMesosTask().getCommand().getShell());
    assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType());
    assertEquals("docker-image", task.getMesosTask().getContainer().getDocker().getImage());
    assertTrue(task.getMesosTask().getContainer().getDocker().getPrivileged());
    assertEquals("/container", task.getMesosTask().getContainer().getVolumes(0).getContainerPath());
    assertEquals("/host", task.getMesosTask().getContainer().getVolumes(0).getHostPath());
    assertEquals(Mode.RW, task.getMesosTask().getContainer().getVolumes(0).getMode());
    Parameter envParameter = Parameter.newBuilder().setKey("env").setValue("var=value").build();
    assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter));
    assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath());
    assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath());
    assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode());
    assertEquals(80, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getContainerPort());
    assertEquals(8080, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getHostPort());
    assertEquals("tcp", task.getMesosTask().getContainer().getDocker().getPortMappings(0).getProtocol());
    assertTrue(MesosUtils.getAllPorts(task.getMesosTask().getResourcesList()).contains(8080L));
    assertEquals(81, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getContainerPort());
    assertEquals(31000, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getHostPort());
    assertEquals("udp", task.getMesosTask().getContainer().getDocker().getPortMappings(1).getProtocol());
    assertEquals(Protos.ContainerInfo.DockerInfo.Network.BRIDGE, task.getMesosTask().getContainer().getDocker().getNetwork());
}

Example 3 with SingularityVolume

use of com.hubspot.mesos.SingularityVolume in project Singularity by HubSpot.

the class SingularityMesosTaskBuilderTest method testMesosContainer.

@Test
public void testMesosContainer() {
    taskResources = new Resources(1, 1, 2);
    final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build();
    final SingularityContainerInfo containerInfo = new SingularityContainerInfo(SingularityContainerType.MESOS, Optional.of(Collections.singletonList(new SingularityVolume("/testing", Optional.of("/host"), SingularityDockerVolumeMode.RW, Optional.of(new SingularityVolumeSource(SingularityVolumeSourceType.DOCKER_VOLUME, Optional.of(new SingularityDockerVolume(Optional.of("rexray"), Optional.of("testvolume-%i"), Collections.singletonMap("iops", "1")))))))), Optional.absent(), Optional.of(new SingularityMesosInfo(Optional.of(new SingularityMesosImage(SingularityMesosImageType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerImage("test:image")), true)))), Optional.of(Arrays.asList(new SingularityNetworkInfo(Optional.of("network-name"), Optional.of(Arrays.asList("blue", "purple")), Optional.of(Arrays.asList(new SingularityPortMapping(0, 8080, Optional.of("tcp")), new SingularityPortMapping(8888, 8081, Optional.of("udp"))))))));
    final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1").setContainerInfo(Optional.of(containerInfo)).build();
    final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(MesosUtils.getPortRangeResource(31010, 31011)), taskRequest, taskResources, executorResources);
    assertEquals(Type.MESOS, task.getMesosTask().getContainer().getType());
    final Image image = task.getMesosTask().getContainer().getMesos().getImage();
    assertEquals(Protos.Image.Type.DOCKER, image.getType());
    assertEquals("test:image", image.getDocker().getName());
    final Volume volume = task.getMesosTask().getContainer().getVolumesList().get(0);
    assertEquals("/testing", volume.getContainerPath());
    assertEquals("/host", volume.getHostPath());
    assertEquals(Volume.Mode.RW, volume.getMode());
    assertEquals(Volume.Source.Type.DOCKER_VOLUME, volume.getSource().getType());
    final DockerVolume dockerVolume = volume.getSource().getDockerVolume();
    assertEquals("rexray", dockerVolume.getDriver());
    assertEquals("testvolume-1", dockerVolume.getName());
    assertEquals("iops", dockerVolume.getDriverOptions().getParameterList().get(0).getKey());
    final NetworkInfo networkInfo = task.getMesosTask().getContainer().getNetworkInfosList().get(0);
    assertEquals("network-name", networkInfo.getName());
    assertEquals(Arrays.asList("blue", "purple"), networkInfo.getGroupsList());
    final List<Protos.NetworkInfo.PortMapping> portMappings = networkInfo.getPortMappingsList();
    assertEquals(2, portMappings.size());
    assertEquals(31010, portMappings.get(0).getHostPort());
    assertEquals(8080, portMappings.get(0).getContainerPort());
    assertEquals("tcp", portMappings.get(0).getProtocol());
    assertEquals(8888, portMappings.get(1).getHostPort());
    assertEquals(8081, portMappings.get(1).getContainerPort());
    assertEquals("udp", portMappings.get(1).getProtocol());
}

Example 4 with SingularityVolume

use of com.hubspot.mesos.SingularityVolume in project Singularity by HubSpot.

the class SingularityMesosTaskBuilder method prepareContainerInfo.

private void prepareContainerInfo(final SingularityOfferHolder offerHolder, final SingularityTaskId taskId, final TaskInfo.Builder bldr, final SingularityContainerInfo containerInfo, final Optional<long[]> ports) {
    ContainerInfo.Builder containerBuilder = ContainerInfo.newBuilder();
    containerBuilder.setType(ContainerInfo.Type.valueOf(containerInfo.getType().toString()));
    final Optional<SingularityDockerInfo> dockerInfo = containerInfo.getDocker();
    if (dockerInfo.isPresent()) {
        final DockerInfo.Builder dockerInfoBuilder = DockerInfo.newBuilder();
        dockerInfoBuilder.setImage(dockerInfo.get().getImage());
        if (dockerInfo.get().getNetwork().isPresent()) {
            dockerInfoBuilder.setNetwork(DockerInfo.Network.valueOf(dockerInfo.get().getNetwork().get().toString()));
        }
        final List<SingularityDockerPortMapping> portMappings = dockerInfo.get().getPortMappings();
        final boolean isBridged = SingularityDockerNetworkType.BRIDGE.equals(dockerInfo.get().getNetwork().orNull());
        if ((dockerInfo.get().hasAllLiteralHostPortMappings() || ports.isPresent()) && !portMappings.isEmpty()) {
            for (SingularityDockerPortMapping singularityDockerPortMapping : portMappings) {
                final Optional<DockerInfo.PortMapping> maybePortMapping = buildPortMapping(singularityDockerPortMapping, ports);
                if (maybePortMapping.isPresent()) {
                    dockerInfoBuilder.addPortMappings(maybePortMapping.get());
                }
            }
        } else if (configuration.getNetworkConfiguration().isDefaultPortMapping() && isBridged && portMappings.isEmpty() && ports.isPresent()) {
            for (long longPort : ports.get()) {
                int port = Ints.checkedCast(longPort);
                dockerInfoBuilder.addPortMappings(DockerInfo.PortMapping.newBuilder().setHostPort(port).setContainerPort(port).build());
            }
        }
        if (!dockerInfo.get().getDockerParameters().isEmpty()) {
            List<Parameter> parameters = new ArrayList<>();
            for (SingularityDockerParameter parameter : dockerInfo.get().getDockerParameters()) {
                parameters.add(Parameter.newBuilder().setKey(parameter.getKey()).setValue(parameter.getValue()).build());
            }
            dockerInfoBuilder.addAllParameters(parameters);
        }
        dockerInfoBuilder.setPrivileged(dockerInfo.get().isPrivileged());
        dockerInfoBuilder.setForcePullImage(dockerInfo.get().isForcePullImage());
        containerBuilder.setDocker(dockerInfoBuilder);
    }
    for (SingularityVolume volumeInfo : containerInfo.getVolumes().or(Collections.<SingularityVolume>emptyList())) {
        final Volume.Builder volumeBuilder = Volume.newBuilder();
        volumeBuilder.setContainerPath(fillInTaskIdValues(volumeInfo.getContainerPath(), offerHolder, taskId));
        if (volumeInfo.getHostPath().isPresent()) {
            volumeBuilder.setHostPath(fillInTaskIdValues(volumeInfo.getHostPath().get(), offerHolder, taskId));
        }
        if (volumeInfo.getMode().isPresent()) {
            volumeBuilder.setMode(Volume.Mode.valueOf(volumeInfo.getMode().get().toString()));
        } else {
            volumeBuilder.setMode(Volume.Mode.RO);
        }
        if (volumeInfo.getSource().isPresent()) {
            final Volume.Source.Builder sourceBuilder = Volume.Source.newBuilder();
            final SingularityVolumeSource source = volumeInfo.getSource().get();
            sourceBuilder.setType(Volume.Source.Type.valueOf(source.getType().toString()));
            if (source.getDockerVolume().isPresent()) {
                final Volume.Source.DockerVolume.Builder dockerVolumeBuilder = Volume.Source.DockerVolume.newBuilder();
                final SingularityDockerVolume dockerVolume = source.getDockerVolume().get();
                if (dockerVolume.getDriver().isPresent()) {
                    dockerVolumeBuilder.setDriver(dockerVolume.getDriver().get());
                }
                if (dockerVolume.getName().isPresent()) {
                    dockerVolumeBuilder.setName(dockerVolume.getName().get().replace("%i", Integer.toString(taskId.getInstanceNo())));
                }
                if (!dockerVolume.getDriverOptions().isEmpty()) {
                    final Parameters.Builder parameters = Parameters.newBuilder();
                    for (Entry<String, String> option : dockerVolume.getDriverOptions().entrySet()) {
                        parameters.addParameter(Parameter.newBuilder().setKey(option.getKey()).setValue(option.getValue()).build());
                    }
                    dockerVolumeBuilder.setDriverOptions(parameters.build());
                }
                sourceBuilder.setDockerVolume(dockerVolumeBuilder.build());
            }
            volumeBuilder.setSource(sourceBuilder.build());
        }
        containerBuilder.addVolumes(volumeBuilder);
    }
    prepareMesosInfo(containerBuilder, containerInfo);
    prepareNetworkInfos(containerBuilder, containerInfo, ports);
    bldr.setContainer(containerBuilder);
}

Example 5 with SingularityVolume

use of com.hubspot.mesos.SingularityVolume in project Singularity by HubSpot.

the class SingularityValidator method checkDeploy.

public SingularityDeploy checkDeploy(SingularityRequest request, SingularityDeploy deploy, List<SingularityTaskId> activeTasks, List<SingularityPendingTaskId> pendingTasks) {
    checkNotNull(request, "request is null");
    checkNotNull(deploy, "deploy is null");
    String deployId = deploy.getId();
    if (deployId == null) {
        checkBadRequest(createDeployIds, "Id must not be null");
        SingularityDeployBuilder builder = deploy.toBuilder();
        builder.setId(createUniqueDeployId());
        deploy = builder.build();
        deployId = deploy.getId();
    }
    checkBadRequest(deployId != null && !DEPLOY_ID_ILLEGAL_PATTERN.matcher(deployId).find(), "Id cannot be null or contain characters other than [a-zA-Z0-9_.]");
    checkBadRequest(deployId.length() <= maxDeployIdSize, "Deploy id must be %s characters or less, it is %s (%s)", maxDeployIdSize, deployId.length(), deployId);
    checkBadRequest(deploy.getRequestId() != null && deploy.getRequestId().equals(request.getId()), "Deploy id must match request id");
    if (request.isLoadBalanced()) {
        checkBadRequest(deploy.getServiceBasePath().isPresent(), "Deploy for loadBalanced request must include serviceBasePath");
        checkBadRequest(deploy.getLoadBalancerGroups().isPresent() && !deploy.getLoadBalancerGroups().get().isEmpty(), "Deploy for a loadBalanced request must include at least one load balacner group");
    }
    checkForIllegalResources(request, deploy);
    if (deploy.getResources().isPresent()) {
        if (deploy.getHealthcheck().isPresent()) {
            HealthcheckOptions healthcheck = deploy.getHealthcheck().get();
            checkBadRequest(!(healthcheck.getPortIndex().isPresent() && healthcheck.getPortNumber().isPresent()), "Can only specify one of portIndex or portNumber for healthchecks");
            if (healthcheck.getPortIndex().isPresent()) {
                checkBadRequest(healthcheck.getPortIndex().get() >= 0, "healthcheckPortIndex cannot be negative");
                checkBadRequest(deploy.getResources().get().getNumPorts() > healthcheck.getPortIndex().get(), String.format("Must request %s ports for healthcheckPortIndex %s, only requested %s", healthcheck.getPortIndex().get() + 1, healthcheck.getPortIndex().get(), deploy.getResources().get().getNumPorts()));
            }
        }
        if (deploy.getLoadBalancerPortIndex().isPresent()) {
            checkBadRequest(deploy.getLoadBalancerPortIndex().get() >= 0, "loadBalancerPortIndex must be greater than 0");
            checkBadRequest(deploy.getResources().get().getNumPorts() > deploy.getLoadBalancerPortIndex().get(), String.format("Must request %s ports for loadBalancerPortIndex %s, only requested %s", deploy.getLoadBalancerPortIndex().get() + 1, deploy.getLoadBalancerPortIndex().get(), deploy.getResources().get().getNumPorts()));
        }
    }
    if (deploy.getHealthcheck().isPresent()) {
        checkBadRequest(!Strings.isNullOrEmpty(deploy.getHealthcheck().get().getUri()), "Must specify a uri when specifying health check parameters");
        if (!deploy.getResources().isPresent() || deploy.getResources().get().getNumPorts() == 0) {
            checkBadRequest(deploy.getHealthcheck().get().getPortNumber().isPresent(), "Either an explicit port number, or port resources and port index must be specified to run healthchecks against a uri");
        }
        if (maxTotalHealthcheckTimeoutSeconds.isPresent()) {
            HealthcheckOptions options = deploy.getHealthcheck().get();
            int intervalSeconds = options.getIntervalSeconds().or(defaultHealthcheckIntervalSeconds);
            int httpTimeoutSeconds = options.getResponseTimeoutSeconds().or(defaultHealthcheckResponseTimeoutSeconds);
            int startupTime = options.getStartupTimeoutSeconds().or(defaultHealthcheckStartupTimeoutSeconds);
            int attempts = options.getMaxRetries().or(defaultHealthcehckMaxRetries) + 1;
            int totalHealthCheckTime = startupTime + ((httpTimeoutSeconds + intervalSeconds) * attempts);
            checkBadRequest(totalHealthCheckTime < maxTotalHealthcheckTimeoutSeconds.get(), String.format("Max healthcheck time cannot be greater than %s, (was startup timeout: %s, interval: %s, attempts: %s)", maxTotalHealthcheckTimeoutSeconds.get(), startupTime, intervalSeconds, attempts));
        }
        if (deploy.getHealthcheck().get().getStartupDelaySeconds().isPresent()) {
            int startUpDelay = deploy.getHealthcheck().get().getStartupDelaySeconds().get();
            checkBadRequest(startUpDelay < defaultKillHealthcheckAfterSeconds, String.format("Health check startup delay time must be less than max health check run time %s (was %s)", defaultKillHealthcheckAfterSeconds, startUpDelay));
        }
    }
    checkBadRequest(deploy.getCommand().isPresent() && !deploy.getExecutorData().isPresent() || deploy.getExecutorData().isPresent() && deploy.getCustomExecutorCmd().isPresent() && !deploy.getCommand().isPresent() || deploy.getContainerInfo().isPresent(), "If not using custom executor, specify a command or containerInfo. If using custom executor, specify executorData and customExecutorCmd and no command.");
    checkBadRequest(!deploy.getContainerInfo().isPresent() || deploy.getContainerInfo().get().getType() != null, "Container type must not be null");
    if (deploy.getLabels().isPresent() && deploy.getMesosTaskLabels().isPresent()) {
        List<SingularityMesosTaskLabel> deprecatedLabels = SingularityMesosTaskLabel.labelsFromMap(deploy.getLabels().get());
        checkBadRequest(deprecatedLabels.containsAll(deploy.getMesosLabels().get()) && deploy.getMesosLabels().get().containsAll(deprecatedLabels), "Can only specify one of 'labels' or 'mesosLabels");
    }
    if (deploy.getTaskLabels().isPresent() && deploy.getMesosTaskLabels().isPresent()) {
        for (Map.Entry<Integer, Map<String, String>> entry : deploy.getTaskLabels().get().entrySet()) {
            List<SingularityMesosTaskLabel> deprecatedLabels = SingularityMesosTaskLabel.labelsFromMap(entry.getValue());
            checkBadRequest(deploy.getMesosTaskLabels().get().containsKey(entry.getKey()) && deprecatedLabels.containsAll(deploy.getMesosTaskLabels().get().get(entry.getKey())) && deploy.getMesosTaskLabels().get().get(entry.getKey()).containsAll(deprecatedLabels), "Can only specify one of 'taskLabels' or 'mesosTaskLabels");
        }
    }
    if (deploy.getContainerInfo().isPresent()) {
        SingularityContainerInfo containerInfo = deploy.getContainerInfo().get();
        checkBadRequest(containerInfo.getType() != null, "container type may not be null");
        if (containerInfo.getVolumes().isPresent() && !containerInfo.getVolumes().get().isEmpty()) {
            for (SingularityVolume volume : containerInfo.getVolumes().get()) {
                checkBadRequest(volume.getContainerPath() != null, "volume containerPath may not be null");
            }
        }
        if (deploy.getContainerInfo().get().getType() == SingularityContainerType.DOCKER) {
            checkDocker(deploy);
        }
    }
    checkBadRequest(deployHistoryHelper.isDeployIdAvailable(request.getId(), deployId), "Can not deploy a deploy that has already been deployed");
    if (deploy.getRunImmediately().isPresent()) {
        deploy = checkImmediateRunDeploy(request, deploy, deploy.getRunImmediately().get(), activeTasks, pendingTasks);
    }
    if (request.isDeployable()) {
        checkRequestForPriorityFreeze(request);
    }
    return deploy;
}