Examples with SingularityAgentUsageWithId com.hubspot.singularity.SingularityAgentUsageWithId used on opensource projects

Example 11 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itCreatesTaskCleanupsWhenAMachineIsOverloadedOnCpu.

@Test
public void itCreatesTaskCleanupsWhenAMachineIsOverloadedOnCpu() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        initRequest();
        initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory());
        saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3)));
        resourceOffers(1);
        SingularityAgentUsage highUsage = new SingularityAgentUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highUsage, "host1"));
        SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0);
        String t1 = taskId1.getId();
        SingularityTaskId taskId2 = taskManager.getActiveTaskIds().get(1);
        String t2 = taskId2.getId();
        SingularityTaskId taskId3 = taskManager.getActiveTaskIds().get(2);
        String t3 = taskId3.getId();
        statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId3).get(), TaskState.TASK_STARTING, Optional.of(taskId3.getStartedAt()));
        // task 1 using 3 cpus
        MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 15, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 1024);
        // task 2 using 2 cpus
        MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 1024);
        // task 3 using 1 cpus
        MesosTaskMonitorObject t3u1 = getTaskMonitor(t3, 5, TimeUnit.MILLISECONDS.toSeconds(taskId3.getStartedAt()) + 5, 1024);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        // First task is cleaned up
        Assertions.assertEquals(TaskCleanupType.REBALANCE_CPU_USAGE, taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType());
        // Second task is not cleaned up because it is from the same request as task 1
        Assertions.assertFalse(taskManager.getTaskCleanup(taskId2.getId()).isPresent());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 12 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itWillConsiderCPUUtilizationForMemoryShuffle.

@Test
public void itWillConsiderCPUUtilizationForMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        String t4id = "test-request-4";
        scheduleTask(t1id, 1, 100000);
        scheduleTask(t2id, 1, 100000);
        scheduleTask(t3id, 1, 100000);
        scheduleTask(t4id, 1, 100000);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent1", "host1"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        SingularityTaskId task4 = taskIdMap.get("host1").get(t4id);
        startTask(task1);
        startTask(task2);
        startTask(task3);
        startTask(task4);
        // not actually necessary to trigger shuffle, but worth leaving in case that changes
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 1000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        MesosTaskMonitorObject t1u1 = getTaskMonitor(task1.getId(), 0.5, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 85000);
        MesosTaskMonitorObject t2u1 = getTaskMonitor(task2.getId(), 2.5, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 4000);
        MesosTaskMonitorObject t3u1 = getTaskMonitor(task3.getId(), 1.5, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 5000);
        MesosTaskMonitorObject t4u1 = getTaskMonitor(task4.getId(), 0.5, TimeUnit.MILLISECONDS.toSeconds(task4.getStartedAt()) + 5, 5000);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1, t4u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 1000, 0, 0, 0, 10, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        // First task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        // Second task is not cleaned up, despite low memory utilization, due to high CPU utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task2.getId()).isPresent());
        // 3rd/4th tasks are cleaned up to reach threshold.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task4.getId()).get().getCleanupType());
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task4.getId()).get().getCleanupType());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 13 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itPrioritizesLowUtilizationTasksForMemoryShuffle.

@Test
public void itPrioritizesLowUtilizationTasksForMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        scheduleTask(t1id, 1, 10);
        scheduleTask(t2id, 1, 10);
        scheduleTask(t3id, 1, 10);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000, 100000, "agent1", "host1"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        startTask(task1);
        startTask(task2);
        startTask(task3);
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 9000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        MesosTaskMonitorObject t1u1 = getTaskMonitor(task1.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 69000);
        MesosTaskMonitorObject t2u1 = getTaskMonitor(task2.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 19000);
        MesosTaskMonitorObject t3u1 = getTaskMonitor(task3.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 10000);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 1000, 0, 0, 0, 10, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        // First task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        // Second task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task2.getId()).isPresent());
        // Third task is cleaned up, due to relatively low utilization, in order to reach desired memory usage.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task3.getId()).get().getCleanupType());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 14 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itWillHandleMultiHostMemoryShuffle.

@Test
public void itWillHandleMultiHostMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        String t4id = "test-request-4";
        scheduleTask(t1id, 1, 100000, 1);
        scheduleTask(t2id, 1, 100000, 1);
        scheduleTask(t3id, 1, 100000, 1);
        scheduleTask(t4id, 1, 100000, 1);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent1", "host1"))).join();
        scheduleTask(t1id, 1, 100000, 2);
        scheduleTask(t2id, 1, 100000, 2);
        scheduleTask(t3id, 1, 100000, 2);
        scheduleTask(t4id, 1, 100000, 2);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent2", "host2"))).join();
        scheduleTask(t1id, 1, 100000, 3);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000, 100000, "agent3", "host3"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        SingularityTaskId task4 = taskIdMap.get("host1").get(t4id);
        for (String host : taskIdMap.keySet()) {
            for (String request : taskIdMap.get(host).keySet()) {
                startTask(taskIdMap.get(host).get(request));
            }
        }
        // not actually necessary to trigger shuffle, but worth leaving in case that changes
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 1000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host2"));
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host3"));
        for (String host : taskIdMap.keySet()) {
            List<MesosTaskMonitorObject> usages = new ArrayList<>();
            for (String request : taskIdMap.get(host).keySet()) {
                SingularityTaskId task = taskIdMap.get(host).get(request);
                MesosTaskMonitorObject usage = null;
                if (request.equals(t1id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 95000);
                } else if (request.equals(t2id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 92000);
                } else if (request.equals(t3id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 90000);
                } else if (request.equals(t4id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task4.getStartedAt()) + 5, 88000);
                }
                usages.add(usage);
            }
            mesosClient.setAgentResourceUsage(host, usages);
        }
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 400000, 0, 10000, 0, 0, 0, 10, 0, 0, 0, 0));
        mesosClient.setAgentMetricsSnapshot("host2", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 400000, 0, 50000, 0, 0, 0, 10, 0, 0, 0, 0));
        mesosClient.setAgentMetricsSnapshot("host3", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 9000, 0, 0, 0, 10, 0, 0, 0, 0));
        // host 1 should shuffle
        // host 2 shouldn't shuffle
        // host 3 should shuffle
        // host 4 should receive shuffled tasks
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        Assertions.assertFalse(taskManager.getTaskCleanup(task2.getId()).isPresent());
        Assertions.assertFalse(taskManager.getTaskCleanup(task3.getId()).isPresent());
        assertMemShuffle(taskIdMap, "host1", t4id);
        assertMemShuffle(taskIdMap, "host3", t1id);
        // should receive shuffled tasks from hosts 1/3
        scheduler.drainPendingQueue();
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent4", "host4"))).join();
        taskIdMap = getTaskIdMapByHostByRequest();
        Assertions.assertNotNull(taskIdMap.get("host4").get(t1id));
        Assertions.assertNotNull(taskIdMap.get("host4").get(t4id));
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 15 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityMesosOfferSchedulerTest method itAccountsForExpectedTaskUsage.

@Test
public void itAccountsForExpectedTaskUsage() {
    initRequest();
    double cpuReserved = 2;
    double memMbReserved = 1000;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(1)));
    resourceOffers(3);
    SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
    String t1 = taskId.getId();
    // 2 cpus used
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, (double) (taskId.getStartedAt() + 5000) / 1000, 1000);
    mesosClient.setAgentResourceUsage("host1", Collections.singletonList(t1u1));
    usagePoller.runActionOnPoll();
    SingularityAgentUsage smallUsage = new SingularityAgentUsage(0.1, 0.1, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 1, 30000, 10, 0, 0, 0, 0, 107374182);
    usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(smallUsage, "host1"));
    usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(smallUsage, "host2"));
    usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(smallUsage, "host3"));
    requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()), SingularityUser.DEFAULT_USER);
    Assertions.assertEquals(2.0, usageManager.getRequestUtilizations().get(requestId).getCpuUsed(), 0.001);
    Offer host2Offer = createOffer(6, 30000, 107374182, "host2", "host2");
    agentAndRackManager.checkOffer(host2Offer);
    Offer host3Offer = createOffer(6, 30000, 107374182, "host3", "host3");
    agentAndRackManager.checkOffer(host3Offer);
    singularityScheduler.drainPendingQueue();
    Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(ImmutableMap.of(host2Offer.getId().getValue(), host2Offer, host3Offer.getId().getValue(), host3Offer), System.currentTimeMillis());
    Assertions.assertEquals(2, offerHolders.size());
    // A single offer should only ever get a single task even though both have room for both tasks here. Adding a task should reduce the score for the next check
    for (SingularityOfferHolder offerHolder : offerHolders) {
        Assertions.assertEquals(1, offerHolder.getAcceptedTasks().size());
    }
}