Examples with SingularityTaskId com.hubspot.singularity.SingularityTaskId used on opensource projects

Example 31 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityUsageTest method itCorrectlyTracksMaxAndMinUtilizedPerRequest.

@Test
public void itCorrectlyTracksMaxAndMinUtilizedPerRequest() {
    initRequest();
    double cpuReserved = 10;
    double memMbReserved = .001;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(2)));
    resourceOffers(1);
    List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds();
    SingularityTaskId t1 = taskIds.get(0);
    SingularityTaskId t2 = taskIds.get(1);
    String host = slaveManager.getObjects().get(0).getHost();
    // used 10 cpu
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1.getId(), 50, getTimestampSeconds(t1, 5), 800);
    // used 8 cpu
    MesosTaskMonitorObject t2u1 = getTaskMonitor(t2.getId(), 40, getTimestampSeconds(t2, 5), 700);
    mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u1, t2u1));
    usagePoller.runActionOnPoll();
    Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent());
    SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get();
    int t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size();
    int t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size();
    Assert.assertEquals(1, t1TaskUsages);
    Assert.assertEquals(1, t2TaskUsages);
    Assert.assertEquals(1, utilization.getRequestUtilizations().size());
    double maxCpu = utilization.getRequestUtilizations().get(0).getMaxCpuUsed();
    double minCpu = utilization.getRequestUtilizations().get(0).getMinCpuUsed();
    long maxMemBytes = utilization.getRequestUtilizations().get(0).getMaxMemBytesUsed();
    long minMemBytes = utilization.getRequestUtilizations().get(0).getMinMemBytesUsed();
    Assert.assertEquals(10, maxCpu, 0);
    Assert.assertEquals(8, minCpu, 0);
    Assert.assertEquals(800, maxMemBytes);
    Assert.assertEquals(700, minMemBytes);
    // new max and min after 2nd run
    // used 12 cpu
    MesosTaskMonitorObject t1u2 = getTaskMonitor(t1.getId(), 110, getTimestampSeconds(t1, 10), 850);
    // used 7 cpu
    MesosTaskMonitorObject t2u2 = getTaskMonitor(t2.getId(), 75, getTimestampSeconds(t2, 10), 600);
    mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u2, t2u2));
    usagePoller.runActionOnPoll();
    Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent());
    utilization = usageManager.getClusterUtilization().get();
    t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size();
    t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size();
    Assert.assertEquals(2, t1TaskUsages);
    Assert.assertEquals(2, t2TaskUsages);
    Assert.assertEquals(1, utilization.getRequestUtilizations().size());
    maxCpu = utilization.getRequestUtilizations().get(0).getMaxCpuUsed();
    minCpu = utilization.getRequestUtilizations().get(0).getMinCpuUsed();
    maxMemBytes = utilization.getRequestUtilizations().get(0).getMaxMemBytesUsed();
    minMemBytes = utilization.getRequestUtilizations().get(0).getMinMemBytesUsed();
    Assert.assertEquals(12, maxCpu, 0);
    Assert.assertEquals(7, minCpu, 0);
    Assert.assertEquals(850, maxMemBytes);
    Assert.assertEquals(600, minMemBytes);
    // same max and min after 3rd run
    // used 8 cpu
    MesosTaskMonitorObject t1u3 = getTaskMonitor(t1.getId(), 150, getTimestampSeconds(t1, 15), 750);
    // used 8 cpu
    MesosTaskMonitorObject t2u3 = getTaskMonitor(t2.getId(), 120, getTimestampSeconds(t2, 15), 700);
    mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u3, t2u3));
    usagePoller.runActionOnPoll();
    Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent());
    utilization = usageManager.getClusterUtilization().get();
    t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size();
    t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size();
    Assert.assertEquals(3, t1TaskUsages);
    Assert.assertEquals(3, t2TaskUsages);
    Assert.assertEquals(1, utilization.getRequestUtilizations().size());
    maxCpu = utilization.getRequestUtilizations().get(0).getMaxCpuUsed();
    minCpu = utilization.getRequestUtilizations().get(0).getMinCpuUsed();
    maxMemBytes = utilization.getRequestUtilizations().get(0).getMaxMemBytesUsed();
    minMemBytes = utilization.getRequestUtilizations().get(0).getMinMemBytesUsed();
    Assert.assertEquals(12, maxCpu, 0);
    Assert.assertEquals(7, minCpu, 0);
    Assert.assertEquals(850, maxMemBytes);
    Assert.assertEquals(600, minMemBytes);
}

Example 32 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityUsageTest method itCreatesTaskCleanupsWhenAMachineIsOverloaded.

@Test
public void itCreatesTaskCleanupsWhenAMachineIsOverloaded() {
    try {
        configuration.setShuffleTasksForOverloadedSlaves(true);
        initRequest();
        initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory());
        saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3)));
        resourceOffers(1);
        SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), Collections.emptyMap(), 1, System.currentTimeMillis(), 1, 30000, 10, 15, 15, 15, 0, 107374182);
        usageManager.saveSpecificSlaveUsageAndSetCurrent("host1", highUsage);
        SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0);
        String t1 = taskId1.getId();
        SingularityTaskId taskId2 = taskManager.getActiveTaskIds().get(1);
        String t2 = taskId2.getId();
        SingularityTaskId taskId3 = taskManager.getActiveTaskIds().get(2);
        String t3 = taskId3.getId();
        statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt()));
        statusUpdate(taskManager.getTask(taskId3).get(), TaskState.TASK_STARTING, Optional.of(taskId3.getStartedAt()));
        // task 1 using 3 cpus
        MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 15, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 1024);
        // task 2 using 2 cpus
        MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 1024);
        // task 3 using 1 cpus
        MesosTaskMonitorObject t3u1 = getTaskMonitor(t3, 5, TimeUnit.MILLISECONDS.toSeconds(taskId3.getStartedAt()) + 5, 1024);
        mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1));
        mesosClient.setSlaveMetricsSnapshot("host1", new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0));
        usagePoller.runActionOnPoll();
        // First task is cleaned up
        Assert.assertEquals(taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType(), TaskCleanupType.REBALANCE_CPU_USAGE);
        // Second task is not cleaned up because it is from the same request as task 1
        Assert.assertFalse(taskManager.getTaskCleanup(taskId2.getId()).isPresent());
    } finally {
        configuration.setShuffleTasksForOverloadedSlaves(false);
    }
}

Example 33 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class ValidatorTest method itForbidsRunNowOfScheduledWhenAlreadyRunning.

@Test(expected = WebApplicationException.class)
public void itForbidsRunNowOfScheduledWhenAlreadyRunning() {
    String deployID = "deploy";
    Optional<String> userEmail = Optional.absent();
    SingularityRequest request = new SingularityRequestBuilder("request2", RequestType.SCHEDULED).setInstances(Optional.of(1)).build();
    Optional<SingularityRunNowRequest> runNowRequest = Optional.absent();
    List<SingularityTaskId> activeTasks = Collections.singletonList(activeTask());
    List<SingularityPendingTaskId> pendingTasks = Collections.emptyList();
    validator.checkRunNowRequest(deployID, userEmail, request, runNowRequest, activeTasks, pendingTasks);
}

Example 34 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class ValidatorTest method whenRunNowItForbidsRequestsForLongRunningTasks.

@Test(expected = WebApplicationException.class)
public void whenRunNowItForbidsRequestsForLongRunningTasks() {
    String deployID = "deploy";
    Optional<String> userEmail = Optional.absent();
    SingularityRequest request = new SingularityRequestBuilder("request2", RequestType.SERVICE).build();
    Optional<SingularityRunNowRequest> runNowRequest = Optional.absent();
    List<SingularityTaskId> activeTasks = Collections.emptyList();
    List<SingularityPendingTaskId> pendingTasks = Collections.emptyList();
    validator.checkRunNowRequest(deployID, userEmail, request, runNowRequest, activeTasks, pendingTasks);
}

Example 35 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class ValidatorTest method whenRunNowIfNoRunIdSetItWillGenerateAnId.

@Test
public void whenRunNowIfNoRunIdSetItWillGenerateAnId() {
    String deployID = "deploy";
    Optional<String> userEmail = Optional.absent();
    SingularityRequest request = new SingularityRequestBuilder("request2", RequestType.ON_DEMAND).build();
    Optional<SingularityRunNowRequest> runNowRequest = Optional.of(runNowRequest());
    List<SingularityTaskId> activeTasks = Collections.emptyList();
    List<SingularityPendingTaskId> pendingTasks = Collections.emptyList();
    SingularityPendingRequest pendingRequest = validator.checkRunNowRequest(deployID, userEmail, request, runNowRequest, activeTasks, pendingTasks);
    Assert.assertTrue(pendingRequest.getRunId().isPresent());
}