Examples with Resources com.hubspot.mesos.Resources used on opensource projects

Example 1 with Resources

use of com.hubspot.mesos.Resources in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method getMaxProbableUsageForSlave.

private MaxProbableUsage getMaxProbableUsageForSlave(List<SingularityTaskId> activeTaskIds, Map<String, RequestUtilization> requestUtilizations, String sanitizedHostname) {
    double cpu = 0;
    double memBytes = 0;
    double diskBytes = 0;
    for (SingularityTaskId taskId : activeTaskIds) {
        if (taskId.getSanitizedHost().equals(sanitizedHostname)) {
            if (requestUtilizations.containsKey(taskId.getRequestId())) {
                RequestUtilization utilization = requestUtilizations.get(taskId.getRequestId());
                // To account for cpu bursts, tend towards max usage if the app is consistently over-utilizing cpu, tend towards avg if it is over-utilized in short bursts
                cpu += (utilization.getMaxCpuUsed() - utilization.getAvgCpuUsed()) * utilization.getCpuBurstRating() + utilization.getAvgCpuUsed();
                memBytes += utilization.getMaxMemBytesUsed();
                diskBytes += utilization.getMaxDiskBytesUsed();
            } else {
                Optional<SingularityTask> maybeTask = taskManager.getTask(taskId);
                if (maybeTask.isPresent()) {
                    Resources resources = maybeTask.get().getTaskRequest().getPendingTask().getResources().or(maybeTask.get().getTaskRequest().getDeploy().getResources()).or(defaultResources);
                    cpu += resources.getCpus();
                    memBytes += resources.getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
                    diskBytes += resources.getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
                }
            }
        }
    }
    return new MaxProbableUsage(cpu, memBytes, diskBytes);
}

Example 2 with Resources

use of com.hubspot.mesos.Resources in project Singularity by HubSpot.

the class SingularitySchedulerTest method testCustomResourcesWithRunNowRequest.

@Test
public void testCustomResourcesWithRunNowRequest() {
    SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND);
    requestResource.postRequest(bldr.build(), singularityUser);
    deploy("d2");
    SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder().setResources(new Resources(2, 2, 0)).build();
    requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest);
    scheduler.drainPendingQueue();
    SingularityPendingTask pendingTaskWithResourcs = taskManager.getPendingTasks().get(0);
    Assert.assertTrue(pendingTaskWithResourcs.getResources().isPresent());
    Assert.assertEquals(pendingTaskWithResourcs.getResources().get().getCpus(), 2, 0.0);
    sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, "slave1", "host1", Optional.of("rack1"))));
    SingularityTask task = taskManager.getActiveTasks().get(0);
    Assert.assertEquals(MesosUtils.getNumCpus(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), Optional.<String>absent()), 2.0, 0.0);
}

Example 3 with Resources

use of com.hubspot.mesos.Resources in project Singularity by HubSpot.

the class SingularitySchedulerTestBase method deployRequest.

protected SingularityDeploy deployRequest(SingularityRequest request, double cpus, double memoryMb) {
    Resources r = new Resources(cpus, memoryMb, 0);
    SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), "deploy1").setCommand(Optional.of("sleep 1")).setResources(Optional.of(r)).build();
    deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent()), singularityUser);
    return deploy;
}

Example 4 with Resources

use of com.hubspot.mesos.Resources in project Singularity by HubSpot.

the class SingularityMesosTaskBuilderTest method testDockerTask.

@Test
public void testDockerTask() {
    taskResources = new Resources(1, 1, 1, 0);
    final Protos.Resource portsResource = Protos.Resource.newBuilder().setName("ports").setType(Protos.Value.Type.RANGES).setRanges(Protos.Value.Ranges.newBuilder().addRange(Protos.Value.Range.newBuilder().setBegin(31000).setEnd(31000).build()).build()).build();
    final SingularityDockerPortMapping literalMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>absent(), 80, Optional.of(SingularityPortMappingType.LITERAL), 8080, Optional.<String>absent());
    final SingularityDockerPortMapping offerMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>absent(), 81, Optional.of(SingularityPortMappingType.FROM_OFFER), 0, Optional.of("udp"));
    final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build();
    final SingularityContainerInfo containerInfo = new SingularityContainerInfo(SingularityContainerType.DOCKER, Optional.of(Arrays.asList(new SingularityVolume("/container", Optional.of("/host"), SingularityDockerVolumeMode.RW), new SingularityVolume("/container/${TASK_REQUEST_ID}/${TASK_DEPLOY_ID}", Optional.of("/host/${TASK_ID}"), SingularityDockerVolumeMode.RO))), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.BRIDGE, Optional.of(Arrays.asList(literalMapping, offerMapping)), Optional.of(false), Optional.<Map<String, String>>of(ImmutableMap.of("env", "var=value")), Optional.absent())));
    final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1").setContainerInfo(Optional.of(containerInfo)).setCommand(Optional.of("/bin/echo")).setArguments(Optional.of(Collections.singletonList("wat"))).build();
    final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources);
    assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue());
    assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount());
    assertEquals("wat", task.getMesosTask().getCommand().getArguments(0));
    assertFalse(task.getMesosTask().getCommand().getShell());
    assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType());
    assertEquals("docker-image", task.getMesosTask().getContainer().getDocker().getImage());
    assertTrue(task.getMesosTask().getContainer().getDocker().getPrivileged());
    assertEquals("/container", task.getMesosTask().getContainer().getVolumes(0).getContainerPath());
    assertEquals("/host", task.getMesosTask().getContainer().getVolumes(0).getHostPath());
    assertEquals(Mode.RW, task.getMesosTask().getContainer().getVolumes(0).getMode());
    Parameter envParameter = Parameter.newBuilder().setKey("env").setValue("var=value").build();
    assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter));
    assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath());
    assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath());
    assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode());
    assertEquals(80, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getContainerPort());
    assertEquals(8080, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getHostPort());
    assertEquals("tcp", task.getMesosTask().getContainer().getDocker().getPortMappings(0).getProtocol());
    assertTrue(MesosUtils.getAllPorts(task.getMesosTask().getResourcesList()).contains(8080L));
    assertEquals(81, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getContainerPort());
    assertEquals(31000, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getHostPort());
    assertEquals("udp", task.getMesosTask().getContainer().getDocker().getPortMappings(1).getProtocol());
    assertEquals(Protos.ContainerInfo.DockerInfo.Network.BRIDGE, task.getMesosTask().getContainer().getDocker().getNetwork());
}

Example 5 with Resources

use of com.hubspot.mesos.Resources in project Singularity by HubSpot.

the class SingularityMesosTaskBuilderTest method testMesosContainer.

@Test
public void testMesosContainer() {
    taskResources = new Resources(1, 1, 2);
    final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build();
    final SingularityContainerInfo containerInfo = new SingularityContainerInfo(SingularityContainerType.MESOS, Optional.of(Collections.singletonList(new SingularityVolume("/testing", Optional.of("/host"), SingularityDockerVolumeMode.RW, Optional.of(new SingularityVolumeSource(SingularityVolumeSourceType.DOCKER_VOLUME, Optional.of(new SingularityDockerVolume(Optional.of("rexray"), Optional.of("testvolume-%i"), Collections.singletonMap("iops", "1")))))))), Optional.absent(), Optional.of(new SingularityMesosInfo(Optional.of(new SingularityMesosImage(SingularityMesosImageType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerImage("test:image")), true)))), Optional.of(Arrays.asList(new SingularityNetworkInfo(Optional.of("network-name"), Optional.of(Arrays.asList("blue", "purple")), Optional.of(Arrays.asList(new SingularityPortMapping(0, 8080, Optional.of("tcp")), new SingularityPortMapping(8888, 8081, Optional.of("udp"))))))));
    final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1").setContainerInfo(Optional.of(containerInfo)).build();
    final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(MesosUtils.getPortRangeResource(31010, 31011)), taskRequest, taskResources, executorResources);
    assertEquals(Type.MESOS, task.getMesosTask().getContainer().getType());
    final Image image = task.getMesosTask().getContainer().getMesos().getImage();
    assertEquals(Protos.Image.Type.DOCKER, image.getType());
    assertEquals("test:image", image.getDocker().getName());
    final Volume volume = task.getMesosTask().getContainer().getVolumesList().get(0);
    assertEquals("/testing", volume.getContainerPath());
    assertEquals("/host", volume.getHostPath());
    assertEquals(Volume.Mode.RW, volume.getMode());
    assertEquals(Volume.Source.Type.DOCKER_VOLUME, volume.getSource().getType());
    final DockerVolume dockerVolume = volume.getSource().getDockerVolume();
    assertEquals("rexray", dockerVolume.getDriver());
    assertEquals("testvolume-1", dockerVolume.getName());
    assertEquals("iops", dockerVolume.getDriverOptions().getParameterList().get(0).getKey());
    final NetworkInfo networkInfo = task.getMesosTask().getContainer().getNetworkInfosList().get(0);
    assertEquals("network-name", networkInfo.getName());
    assertEquals(Arrays.asList("blue", "purple"), networkInfo.getGroupsList());
    final List<Protos.NetworkInfo.PortMapping> portMappings = networkInfo.getPortMappingsList();
    assertEquals(2, portMappings.size());
    assertEquals(31010, portMappings.get(0).getHostPort());
    assertEquals(8080, portMappings.get(0).getContainerPort());
    assertEquals("tcp", portMappings.get(0).getProtocol());
    assertEquals(8888, portMappings.get(1).getHostPort());
    assertEquals(8081, portMappings.get(1).getContainerPort());
    assertEquals("udp", portMappings.get(1).getProtocol());
}