Examples with SingularityAgentUsageWithId com.hubspot.singularity.SingularityAgentUsageWithId used on opensource projects

Example 6 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityMesosOfferSchedulerTest method itAccountsForMaxHistoricalTaskUsage.

@Test
public void itAccountsForMaxHistoricalTaskUsage() {
    initRequest();
    double cpuReserved = 2;
    double memMbReserved = 1000;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(1)));
    resourceOffers(3);
    SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
    String t1 = taskId.getId();
    // 2 cpus used
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, getTimestampSeconds(taskId, 5), 1000);
    mesosClient.setAgentResourceUsage("host1", Collections.singletonList(t1u1));
    usagePoller.runActionOnPoll();
    // 1 cpus used
    MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 11, getTimestampSeconds(taskId, 6), 1000);
    mesosClient.setAgentResourceUsage("host1", Collections.singletonList(t1u2));
    usagePoller.runActionOnPoll();
    SingularityAgentUsage smallUsage = new SingularityAgentUsage(0.1, 0.1, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 1, 30000, 10, 0, 0, 0, 0, 107374182);
    usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(smallUsage, "host1"));
    usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(smallUsage, "host2"));
    usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(smallUsage, "host3"));
    requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()), SingularityUser.DEFAULT_USER);
    Assertions.assertEquals(3.0, usageManager.getRequestUtilizations().get(requestId).getCpuUsed(), 0.001);
    Offer host2Offer = createOffer(6, 30000, 107374182, "host2", "host2");
    agentAndRackManager.checkOffer(host2Offer);
    Offer host3Offer = createOffer(6, 30000, 107374182, "host3", "host3");
    agentAndRackManager.checkOffer(host3Offer);
    singularityScheduler.drainPendingQueue();
    Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(ImmutableMap.of(host2Offer.getId().getValue(), host2Offer, host3Offer.getId().getValue(), host3Offer), System.currentTimeMillis());
    Assertions.assertEquals(2, offerHolders.size());
    // A single offer should only ever get a single task even though both have room for both tasks here. Adding a task should reduce the score for the next check
    for (SingularityOfferHolder offerHolder : offerHolders) {
        Assertions.assertEquals(1, offerHolder.getAcceptedTasks().size());
    }
}

Example 7 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itDelaysTaskShuffles.

@Test
public void itDelaysTaskShuffles() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(15);
        initRequest();
        initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory());
        saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3)));
        resourceOffers(1);
        SingularityAgentUsage highUsage = new SingularityAgentUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highUsage, "host1"));
        SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0);
        String t1 = taskId1.getId();
        SingularityTaskId taskId2 = taskManager.getActiveTaskIds().get(1);
        String t2 = taskId2.getId();
        SingularityTaskId taskId3 = taskManager.getActiveTaskIds().get(2);
        String t3 = taskId3.getId();
        statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId3).get(), TaskState.TASK_STARTING, Optional.of(taskId3.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_RUNNING, Optional.of(taskId2.getStartedAt() - TimeUnit.MINUTES.toMillis(15)));
        // task 1 using 3 cpus
        MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 15, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 1024);
        // task 2 using 2 cpus
        MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 1024);
        // task 3 using 1 cpus
        MesosTaskMonitorObject t3u1 = getTaskMonitor(t3, 5, TimeUnit.MILLISECONDS.toSeconds(taskId3.getStartedAt()) + 5, 1024);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        // Tasks are not cleaned up because they haven't been running for long enough.
        Assertions.assertFalse(taskManager.getTaskCleanup(taskId1.getId()).isPresent());
        Assertions.assertFalse(taskManager.getTaskCleanup(taskId3.getId()).isPresent());
        // Even though it's not the worst offender, task 2 is cleaned up because it's been running long enough.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_CPU_USAGE, taskManager.getTaskCleanup(taskId2.getId()).get().getCleanupType());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 8 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itWillShuffleToDesiredMemoryThresholdIfPossibleForMemoryShuffle.

@Test
public void itWillShuffleToDesiredMemoryThresholdIfPossibleForMemoryShuffle() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShufflePerHost(2);
        configuration.setMaxTasksToShuffleTotal(5);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        String t1id = "test-request-1";
        String t2id = "test-request-2";
        String t3id = "test-request-3";
        String t4id = "test-request-4";
        scheduleTask(t1id, 1, 10);
        scheduleTask(t2id, 1, 10);
        scheduleTask(t3id, 1, 10);
        scheduleTask(t4id, 1, 10);
        sms.resourceOffers(ImmutableList.of(createOffer(10, 100000, 100000, "agent1", "host1"))).join();
        System.out.println(taskManager.getActiveTaskIds());
        Map<String, Map<String, SingularityTaskId>> taskIdMap = getTaskIdMapByHostByRequest();
        SingularityTaskId task1 = taskIdMap.get("host1").get(t1id);
        SingularityTaskId task2 = taskIdMap.get("host1").get(t2id);
        SingularityTaskId task3 = taskIdMap.get("host1").get(t3id);
        SingularityTaskId task4 = taskIdMap.get("host1").get(t4id);
        startTask(task1);
        startTask(task2);
        startTask(task3);
        startTask(task4);
        // not actually necessary to trigger shuffle, but worth leaving in case that changes
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(1, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 100000, 1000, 10, 10, 10, 10, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        MesosTaskMonitorObject t1u1 = getTaskMonitor(task1.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task1.getStartedAt()) + 5, 87000);
        MesosTaskMonitorObject t2u1 = getTaskMonitor(task2.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task2.getStartedAt()) + 5, 9000);
        MesosTaskMonitorObject t3u1 = getTaskMonitor(task3.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task3.getStartedAt()) + 5, 1100);
        MesosTaskMonitorObject t4u1 = getTaskMonitor(task4.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(task4.getStartedAt()) + 5, 1000);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1, t4u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 100000, 0, 1000, 0, 0, 0, 10, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        // First task is not cleaned up, due to relatively high utilization.
        Assertions.assertFalse(taskManager.getTaskCleanup(task1.getId()).isPresent());
        // Second task is cleaned up, in order to reach desired memory threshold.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task2.getId()).get().getCleanupType());
        // Third task is not cleaned up, as doing so will not reach the desired memory threshold given available shuffles.
        Assertions.assertFalse(taskManager.getTaskCleanup(task3.getId()).isPresent());
        // Fourth task is cleaned up, as it has the lowest utilization.
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(task4.getId()).get().getCleanupType());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 9 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itPrioritizesMemoryShuffleOverCpu.

@Test
public void itPrioritizesMemoryShuffleOverCpu() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShuffleTotal(1);
        configuration.setShuffleTasksWhenAgentMemoryUtilizationPercentageExceeds(0.90);
        initRequest();
        initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory());
        saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(2)));
        sms.resourceOffers(ImmutableList.of(createOffer(1, 128, 50000, "agent1", "host1"), createOffer(1, 128, 50000, "agent2", "host2"))).join();
        SingularityAgentUsage highMemUsage = new SingularityAgentUsage(10, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 10000, 10, 10, 10, 10, 0, 107374182);
        SingularityAgentUsage highCpuUsage = new SingularityAgentUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highMemUsage, "host1"));
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highCpuUsage, "host2"));
        SingularityTaskId host1Task = null;
        SingularityTaskId host2Task = null;
        System.out.println(taskManager.getActiveTaskIds());
        for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) {
            if (taskId.getSanitizedHost().equals("host1")) {
                host1Task = taskId;
            } else if (taskId.getSanitizedHost().equals("host2")) {
                host2Task = taskId;
            }
        }
        statusUpdate(taskManager.getTask(host1Task).get(), TaskState.TASK_STARTING, Optional.of(host1Task.getStartedAt()));
        statusUpdate(taskManager.getTask(host2Task).get(), TaskState.TASK_STARTING, Optional.of(host2Task.getStartedAt()));
        // task 1 using 3G mem
        MesosTaskMonitorObject t1u1 = getTaskMonitor(host1Task.getId(), 2, TimeUnit.MILLISECONDS.toSeconds(host1Task.getStartedAt()) + 5, 199000);
        // task 2 using 2G mem
        MesosTaskMonitorObject t2u1 = getTaskMonitor(host2Task.getId(), 5, TimeUnit.MILLISECONDS.toSeconds(host2Task.getStartedAt()) + 5, 63333);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 200000, 0, 1000, 0, 0, 0, 10, 0, 0, 0, 0));
        mesosClient.setAgentResourceUsage("host2", Arrays.asList(t2u1));
        mesosClient.setAgentMetricsSnapshot("host2", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        System.out.println(taskManager.getCleanupTaskIds().toString());
        // First task is cleaned up
        Assertions.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(host1Task.getId()).get().getCleanupType());
        // Second task is not cleaned up because it is a cpu shuffle
        Assertions.assertFalse(taskManager.getTaskCleanup(host2Task.getId()).isPresent());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
    }
}

Example 10 with SingularityAgentUsageWithId

use of com.hubspot.singularity.SingularityAgentUsageWithId in project Singularity by HubSpot.

the class SingularityUsageTest method itLimitsTheNumberOfTaskCleanupsToCreate.

@Test
public void itLimitsTheNumberOfTaskCleanupsToCreate() {
    try {
        configuration.setShuffleTasksForOverloadedAgents(true);
        configuration.setMinutesBeforeNewTaskEligibleForShuffle(0);
        configuration.setMaxTasksToShuffleTotal(1);
        initRequest();
        initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory());
        saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3)));
        resourceOffers(1);
        SingularityAgentUsage highUsage = new SingularityAgentUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182);
        usageManager.saveCurrentAgentUsage(new SingularityAgentUsageWithId(highUsage, "host1"));
        SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0);
        String t1 = taskId1.getId();
        SingularityTaskId taskId2 = taskManager.getActiveTaskIds().get(1);
        String t2 = taskId2.getId();
        statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt()));
        // task 1 using 3 cpus
        MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 15, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 1024);
        // task 2 using 2 cpus
        MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 1024);
        mesosClient.setAgentResourceUsage("host1", Arrays.asList(t1u1, t2u1));
        mesosClient.setAgentMetricsSnapshot("host1", new MesosAgentMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        // First task is cleaned up
        Assertions.assertEquals(TaskCleanupType.REBALANCE_CPU_USAGE, taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType());
        // Second task doesn't get cleaned up dur to cluster wide limit
        Assertions.assertFalse(taskManager.getTaskCleanup(taskId2.getId()).isPresent());
    } finally {
        configuration.setShuffleTasksForOverloadedAgents(false);
        configuration.setMaxTasksToShuffleTotal(6);
    }
}