Examples with MesosAgentMetricsSnapshotObject com.hubspot.mesos.json.MesosAgentMetricsSnapshotObject used on opensource projects

Example 11 with MesosAgentMetricsSnapshotObject

use of com.hubspot.mesos.json.MesosAgentMetricsSnapshotObject in project Singularity by HubSpot.

the class SingularityUsageTest method itWillConsiderCPUUtilizationForMemoryShuffle.

@Test
public void itWillConsiderCPUUtilizationForMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        String t4id = "test-request-4";
        scheduleTask(t1id, 1, 100000);
        scheduleTask(t2id, 1, 100000);
        scheduleTask(t3id, 1, 100000);
        scheduleTask(t4id, 1, 100000);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent1", "host1"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        SingularityTaskId task4 = taskIdMap.get("host1").get(t4id);
        startTask(task1);
        startTask(task2);
        startTask(task3);
        startTask(task4);
        // not actually necessary to trigger shuffle, but worth leaving in case that changes
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 1000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        MesosTaskMonitorObject t1u1 = getTaskMonitor(task1.getId(), 0.5, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 85000);
        MesosTaskMonitorObject t2u1 = getTaskMonitor(task2.getId(), 2.5, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 4000);
        MesosTaskMonitorObject t3u1 = getTaskMonitor(task3.getId(), 1.5, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 5000);
        MesosTaskMonitorObject t4u1 = getTaskMonitor(task4.getId(), 0.5, TimeUnit.MILLISECONDS.toSeconds(task4.getStartedAt()) + 5, 5000);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1, t4u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 1000, 0, 0, 0, 10, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        // First task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        // Second task is not cleaned up, despite low memory utilization, due to high CPU utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task2.getId()).isPresent());
        // 3rd/4th tasks are cleaned up to reach threshold.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task4.getId()).get().getCleanupType());
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task4.getId()).get().getCleanupType());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 12 with MesosAgentMetricsSnapshotObject

use of com.hubspot.mesos.json.MesosAgentMetricsSnapshotObject in project Singularity by HubSpot.

the class SingularityUsageTest method itPrioritizesLowUtilizationTasksForMemoryShuffle.

@Test
public void itPrioritizesLowUtilizationTasksForMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        scheduleTask(t1id, 1, 10);
        scheduleTask(t2id, 1, 10);
        scheduleTask(t3id, 1, 10);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000, 100000, "agent1", "host1"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        startTask(task1);
        startTask(task2);
        startTask(task3);
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 9000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        MesosTaskMonitorObject t1u1 = getTaskMonitor(task1.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 69000);
        MesosTaskMonitorObject t2u1 = getTaskMonitor(task2.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 19000);
        MesosTaskMonitorObject t3u1 = getTaskMonitor(task3.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 10000);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 1000, 0, 0, 0, 10, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        // First task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        // Second task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task2.getId()).isPresent());
        // Third task is cleaned up, due to relatively low utilization, in order to reach desired memory usage.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task3.getId()).get().getCleanupType());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 13 with MesosAgentMetricsSnapshotObject

use of com.hubspot.mesos.json.MesosAgentMetricsSnapshotObject in project Singularity by HubSpot.

the class SingularityUsageTest method itWillHandleMultiHostMemoryShuffle.

@Test
public void itWillHandleMultiHostMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        String t4id = "test-request-4";
        scheduleTask(t1id, 1, 100000, 1);
        scheduleTask(t2id, 1, 100000, 1);
        scheduleTask(t3id, 1, 100000, 1);
        scheduleTask(t4id, 1, 100000, 1);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent1", "host1"))).join();
        scheduleTask(t1id, 1, 100000, 2);
        scheduleTask(t2id, 1, 100000, 2);
        scheduleTask(t3id, 1, 100000, 2);
        scheduleTask(t4id, 1, 100000, 2);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent2", "host2"))).join();
        scheduleTask(t1id, 1, 100000, 3);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000, 100000, "agent3", "host3"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        SingularityTaskId task4 = taskIdMap.get("host1").get(t4id);
        for (String host : taskIdMap.keySet()) {
            for (String request : taskIdMap.get(host).keySet()) {
                startTask(taskIdMap.get(host).get(request));
            }
        }
        // not actually necessary to trigger shuffle, but worth leaving in case that changes
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 1000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host2"));
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host3"));
        for (String host : taskIdMap.keySet()) {
            List<MesosTaskMonitorObject> usages = new ArrayList<>();
            for (String request : taskIdMap.get(host).keySet()) {
                SingularityTaskId task = taskIdMap.get(host).get(request);
                MesosTaskMonitorObject usage = null;
                if (request.equals(t1id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 95000);
                } else if (request.equals(t2id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 92000);
                } else if (request.equals(t3id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 90000);
                } else if (request.equals(t4id)) {
                    usage = getTaskMonitor(task.getId(), 1, TimeUnit.MILLISECONDS.toSeconds(task4.getStartedAt()) + 5, 88000);
                }
                usages.add(usage);
            }
            mesosClient.setAgentResourceUsage(host, usages);
        }
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 400000, 0, 10000, 0, 0, 0, 10, 0, 0, 0, 0));
        mesosClient.setAgentMetricsSnapshot("host2", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 400000, 0, 50000, 0, 0, 0, 10, 0, 0, 0, 0));
        mesosClient.setAgentMetricsSnapshot("host3", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 1, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 9000, 0, 0, 0, 10, 0, 0, 0, 0));
        // host 1 should shuffle
        // host 2 shouldn't shuffle
        // host 3 should shuffle
        // host 4 should receive shuffled tasks
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        Assertions.assertFalse(taskManager.getTaskCleanup(task2.getId()).isPresent());
        Assertions.assertFalse(taskManager.getTaskCleanup(task3.getId()).isPresent());
        assertMemShuffle(taskIdMap, "host1", t4id);
        assertMemShuffle(taskIdMap, "host3", t1id);
        // should receive shuffled tasks from hosts 1/3
        scheduler.drainPendingQueue();
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000 * 4, 100000, "agent4", "host4"))).join();
        taskIdMap = getTaskIdMapByHostByRequest();
        Assertions.assertNotNull(taskIdMap.get("host4").get(t1id));
        Assertions.assertNotNull(taskIdMap.get("host4").get(t4id));
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}