Examples with MesosTaskMonitorObject com.hubspot.mesos.json.MesosTaskMonitorObject used on opensource projects

Example 6 with MesosTaskMonitorObject

use of com.hubspot.mesos.json.MesosTaskMonitorObject in project Singularity by HubSpot.

the class SingularityMesosOfferSchedulerTest method itAccountsForExpectedTaskUsage.

@Test
public void itAccountsForExpectedTaskUsage() {
    initRequest();
    double cpuReserved = 2;
    double memMbReserved = 1000;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(1)));
    resourceOffers(3);
    SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
    String t1 = taskId.getId();
    // 2 cpus used
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 5, 1000);
    mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u1));
    usagePoller.runActionOnPoll();
    Map<ResourceUsageType, Number> longRunningTasksUsage = new HashMap<>();
    longRunningTasksUsage.put(ResourceUsageType.CPU_USED, 0.1);
    longRunningTasksUsage.put(ResourceUsageType.MEMORY_BYTES_USED, 0.1);
    longRunningTasksUsage.put(ResourceUsageType.DISK_BYTES_USED, 0.1);
    SingularitySlaveUsage smallUsage = new SingularitySlaveUsage(0.1, 0.1, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), longRunningTasksUsage, 1, System.currentTimeMillis(), 1, 30000, 10, 0, 0, 0, 0, 107374182);
    usageManager.saveSpecificSlaveUsageAndSetCurrent("host1", smallUsage);
    usageManager.saveSpecificSlaveUsageAndSetCurrent("host2", smallUsage);
    usageManager.saveSpecificSlaveUsageAndSetCurrent("host3", smallUsage);
    requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), SingularityUser.DEFAULT_USER);
    Assert.assertEquals(2.0, usageManager.getRequestUtilizations().get(requestId).getCpuUsed(), 0.001);
    Offer host2Offer = createOffer(6, 30000, 107374182, "host2", "host2");
    slaveAndRackManager.checkOffer(host2Offer);
    Offer host3Offer = createOffer(6, 30000, 107374182, "host3", "host3");
    slaveAndRackManager.checkOffer(host3Offer);
    Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(Arrays.asList(host2Offer, host3Offer));
    Assert.assertEquals(2, offerHolders.size());
    // A single offer should only ever get a single task even though both have room for both tasks here. Adding a task should reduce the score for the next check
    for (SingularityOfferHolder offerHolder : offerHolders) {
        Assert.assertEquals(1, offerHolder.getAcceptedTasks().size());
    }
}

Example 7 with MesosTaskMonitorObject

use of com.hubspot.mesos.json.MesosTaskMonitorObject in project Singularity by HubSpot.

the class SingularityUsageTest method itTracksOverusedCpuInClusterUtilization.

@Test
public void itTracksOverusedCpuInClusterUtilization() {
    initRequest();
    double cpuReserved = 2;
    double memMbReserved = .0009;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(1)));
    resourceOffers(1);
    SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
    String t1 = taskId.getId();
    String host = slaveManager.getObjects().get(0).getHost();
    // 4 cpus used
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 20, getTimestampSeconds(taskId, 5), 1024);
    mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u1));
    usagePoller.runActionOnPoll();
    // 4 cpus used
    MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 40, getTimestampSeconds(taskId, 10), 1024);
    mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u2));
    usagePoller.runActionOnPoll();
    Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent());
    SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get();
    int taskUsages = usageManager.getTaskUsage(t1).size();
    testUtilization(utilization, 2, taskUsages, cpuReserved, memMbReserved, 1, 0, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0);
    Assert.assertEquals(requestId, utilization.getMaxOverUtilizedCpuRequestId());
}

Example 8 with MesosTaskMonitorObject

use of com.hubspot.mesos.json.MesosTaskMonitorObject in project Singularity by HubSpot.

the class SingularityUsageTest method itDoesntIncludePerfectlyUtilizedRequestsInClusterUtilization.

@Test
public void itDoesntIncludePerfectlyUtilizedRequestsInClusterUtilization() {
    initRequest();
    double cpuReserved = 2;
    double memMbReserved = .001;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(1)));
    resourceOffers(1);
    SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
    String t1 = taskId.getId();
    String host = slaveManager.getObjects().get(0).getHost();
    // 2 cpus used
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, getTimestampSeconds(taskId, 5), 1024);
    mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u1));
    usagePoller.runActionOnPoll();
    // 2 cpus used
    MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 20, getTimestampSeconds(taskId, 10), 900);
    mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u2));
    usagePoller.runActionOnPoll();
    Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent());
    SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get();
    int taskUsages = usageManager.getTaskUsage(t1).size();
    testUtilization(utilization, 2, taskUsages, cpuReserved, memMbReserved, 0, 0, 1, 0, 0, 86, 0, 0, 86, 0, 0, 86);
    Assert.assertEquals(requestId, utilization.getMaxUnderUtilizedMemBytesRequestId());
}

Example 9 with MesosTaskMonitorObject

use of com.hubspot.mesos.json.MesosTaskMonitorObject in project Singularity by HubSpot.

the class SingularityUsageTest method testUsagePollerSimple.

@Test
public void testUsagePollerSimple() {
    // works with no slaves
    usagePoller.runActionOnPoll();
    cleaner.runActionOnPoll();
    initRequest();
    initFirstDeploy();
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(1)));
    resourceOffers(1);
    SingularityTask firstTask = taskManager.getActiveTasks().get(0);
    String hostname = firstTask.getHostname();
    MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100);
    mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage));
    usagePoller.runActionOnPoll();
    String slaveId = firstTask.getAgentId().getValue();
    List<String> slaves = usageManager.getSlavesWithUsage();
    Assert.assertEquals(1, slaves.size());
    Assert.assertEquals(slaves.get(0), slaveId);
    Assert.assertEquals(0, usageManager.getSlaveUsage(slaveId).get(0).getCpusUsed(), 0);
    Assert.assertEquals(100, usageManager.getSlaveUsage(slaveId).get(0).getMemoryBytesUsed());
    SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0);
    Assert.assertEquals(2, first.getCpuSeconds(), 0);
    Assert.assertEquals(100, first.getMemoryTotalBytes(), 0);
    Assert.assertEquals(5, first.getTimestamp(), 0);
}

Example 10 with MesosTaskMonitorObject

use of com.hubspot.mesos.json.MesosTaskMonitorObject in project Singularity by HubSpot.

the class SingularityUsageTest method testUsageCleaner.

@Test
public void testUsageCleaner() {
    initRequest();
    initFirstDeploy();
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(2)));
    resourceOffers(1);
    List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds();
    String t1 = taskIds.get(0).getId();
    String t2 = taskIds.get(1).getId();
    String slaveId = slaveManager.getObjectIds().get(0);
    String host = slaveManager.getObjects().get(0).getHost();
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 2, 5, 100);
    MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, 5, 1000);
    mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u1, t2u1));
    usagePoller.runActionOnPoll();
    cleaner.runActionOnPoll();
    Assert.assertEquals(2, usageManager.getTasksWithUsage().size());
    Assert.assertEquals(1, usageManager.getSlavesWithUsage().size());
    Assert.assertEquals(1100, usageManager.getAllCurrentSlaveUsage().get(0).getMemoryBytesUsed());
    // kill task one
    statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_KILLED);
    killKilledTasks();
    cleaner.runActionOnPoll();
    Assert.assertEquals(1, usageManager.getTasksWithUsage().size());
    Assert.assertEquals(1, usageManager.getSlavesWithUsage().size());
    slaveManager.changeState(slaveId, MachineState.DEAD, Optional.absent(), Optional.absent());
    cleaner.runActionOnPoll();
    Assert.assertEquals(1, usageManager.getTasksWithUsage().size());
    Assert.assertEquals(0, usageManager.getSlavesWithUsage().size());
}