Examples with SingularityKilledTaskIdRecord com.hubspot.singularity.SingularityKilledTaskIdRecord used on opensource projects

Example 1 with SingularityKilledTaskIdRecord

use of com.hubspot.singularity.SingularityKilledTaskIdRecord in project Singularity by HubSpot.

the class SingularityCleaner method checkKilledTaskIdRecords.

private void checkKilledTaskIdRecords() {
    final long start = System.currentTimeMillis();
    final List<SingularityKilledTaskIdRecord> killedTaskIdRecords = taskManager.getKilledTaskIdRecords();
    if (killedTaskIdRecords.isEmpty()) {
        LOG.trace("No killed taskId records");
        return;
    }
    AtomicInteger obsolete = new AtomicInteger(0);
    AtomicInteger waiting = new AtomicInteger(0);
    AtomicInteger rekilled = new AtomicInteger(0);
    CompletableFutures.allOf(killedTaskIdRecords.stream().collect(Collectors.groupingBy(record -> record.getTaskId().getRequestId())).entrySet().stream().map(killedTaskIdRecordsForRequest -> CompletableFuture.runAsync(() -> lock.runWithRequestLock(() -> {
        for (SingularityKilledTaskIdRecord killedTaskIdRecord : killedTaskIdRecordsForRequest.getValue()) {
            if (!taskManager.isActiveTask(killedTaskIdRecord.getTaskId())) {
                SingularityDeleteResult deleteResult = taskManager.deleteKilledRecord(killedTaskIdRecord.getTaskId());
                LOG.debug("Deleting obsolete {} - {}", killedTaskIdRecord, deleteResult);
                obsolete.getAndIncrement();
                continue;
            }
            long duration = start - killedTaskIdRecord.getTimestamp();
            if (duration > configuration.getAskDriverToKillTasksAgainAfterMillis()) {
                LOG.info("{} is still active, and time since last kill {} is greater than configured (askDriverToKillTasksAgainAfterMillis) {} - asking driver to kill again", killedTaskIdRecord, JavaUtils.durationFromMillis(duration), JavaUtils.durationFromMillis(configuration.getAskDriverToKillTasksAgainAfterMillis()));
                scheduler.killAndRecord(killedTaskIdRecord.getTaskId(), killedTaskIdRecord.getRequestCleanupType(), killedTaskIdRecord.getTaskCleanupType(), Optional.of(killedTaskIdRecord.getOriginalTimestamp()), Optional.of(killedTaskIdRecord.getRetries()), Optional.empty());
                rekilled.getAndIncrement();
            } else {
                LOG.trace("Ignoring {}, because duration {} is less than configured (askDriverToKillTasksAgainAfterMillis) {}", killedTaskIdRecord, JavaUtils.durationFromMillis(duration), JavaUtils.durationFromMillis(configuration.getAskDriverToKillTasksAgainAfterMillis()));
                waiting.getAndIncrement();
            }
        }
    }, killedTaskIdRecordsForRequest.getKey(), String.format("%s#%s", getClass().getSimpleName(), "checkKilledTaskIdRecords")), cleanerExecutor)).collect(Collectors.toList())).join();
    LOG.info("{} obsolete, {} waiting, {} rekilled tasks based on {} killedTaskIdRecords", obsolete, waiting, rekilled, killedTaskIdRecords.size());
}

Example 2 with SingularityKilledTaskIdRecord

use of com.hubspot.singularity.SingularityKilledTaskIdRecord in project Singularity by HubSpot.

the class SingularityMesosSchedulerImpl method killAndRecord.

public void killAndRecord(SingularityTaskId taskId, Optional<RequestCleanupType> requestCleanupType, Optional<TaskCleanupType> taskCleanupType, Optional<Long> originalTimestamp, Optional<Integer> retries, Optional<String> user) {
    Preconditions.checkState(isRunning());
    Optional<TaskCleanupType> maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType);
    if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) {
        Optional<SingularityTask> task = taskManager.getTask(taskId);
        if (task.isPresent()) {
            if (task.get().getTaskRequest().getDeploy().getCustomExecutorCmd().isPresent()) {
                byte[] messageBytes = transcoder.toBytes(new SingularityTaskDestroyFrameworkMessage(taskId, user));
                mesosSchedulerClient.frameworkMessage(MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()), MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()), messageBytes);
            } else {
                LOG.warn("Not using custom executor, will not send framework message to destroy task");
            }
        } else {
            String message = String.format("No task data available to build kill task framework message for task %s", taskId);
            exceptionNotifier.notify(message);
            LOG.error(message);
        }
    }
    mesosSchedulerClient.kill(TaskID.newBuilder().setValue(taskId.toString()).build());
    taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskId, System.currentTimeMillis(), originalTimestamp.orElse(System.currentTimeMillis()), requestCleanupType, taskCleanupType, retries.orElse(-1) + 1));
}

Example 3 with SingularityKilledTaskIdRecord

use of com.hubspot.singularity.SingularityKilledTaskIdRecord in project Singularity by HubSpot.

the class SingularitySchedulerTest method testTaskLaunchesInRackSensitiveWithKillingTask.

@Test
public void testTaskLaunchesInRackSensitiveWithKillingTask() {
    try {
        configuration.setExpectedRacksCount(Optional.of(3));
        // Set up hosts + racks
        sms.resourceOffers(Arrays.asList(createOffer(1, 1, 1, "agent1", "host1", Optional.of("rack1")), createOffer(1, 1, 1, "agent2", "host2", Optional.of("rack2")), createOffer(1, 1, 1, "agent3", "host3", Optional.of("rack3")))).join();
        initRequest();
        initFirstDeploy();
        requestResource.postRequest(request.toBuilder().setInstances(Optional.of(3)).setRackSensitive(Optional.of(true)).setAgentPlacement(Optional.of(AgentPlacement.SEPARATE)).build(), SingularityUser.DEFAULT_USER);
        schedulerPoller.runActionOnPoll();
        Assertions.assertEquals(3, taskManager.getPendingTaskIds().size());
        sms.resourceOffers(Arrays.asList(createOffer(2, 2000, 2000, "agent1", "host1", Optional.of("rack1")), createOffer(2, 2000, 2000, "agent2", "host2", Optional.of("rack2")), createOffer(2, 2000, 2000, "agent3", "host3", Optional.of("rack3")))).join();
        Assertions.assertEquals(3, taskManager.getActiveTaskIds().size());
        SingularityTaskId taskTwo = taskManager.getActiveTaskIds().stream().filter(t -> t.getInstanceNo() == 2).findFirst().get();
        SingularityTaskId taskThree = taskManager.getActiveTaskIds().stream().filter(t -> t.getInstanceNo() == 3).findFirst().get();
        requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).setRackSensitive(Optional.of(true)).setAgentPlacement(Optional.of(AgentPlacement.SEPARATE)).build(), SingularityUser.DEFAULT_USER);
        taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.empty(), TaskCleanupType.USER_REQUESTED, System.currentTimeMillis(), taskTwo, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()));
        taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.empty(), TaskCleanupType.USER_REQUESTED, System.currentTimeMillis(), taskThree, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()));
        cleaner.drainCleanupQueue();
        schedulerPoller.runActionOnPoll();
        // Mimic tasks that have been killed (and their cleanup removed) but are slow to shut down
        taskManager.deleteCleanupTask(taskTwo.toString());
        taskManager.deleteCleanupTask(taskThree.toString());
        taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskTwo, System.currentTimeMillis(), System.currentTimeMillis(), Optional.empty(), Optional.empty(), 1));
        taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskTwo, System.currentTimeMillis(), System.currentTimeMillis(), Optional.empty(), Optional.empty(), 1));
        Assertions.assertEquals(3, taskManager.getActiveTaskIds().size());
        Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
        sms.resourceOffers(Arrays.asList(createOffer(2, 2000, 2000, "agent4", "host4", Optional.of("rack1")), createOffer(2, 2000, 2000, "agent5", "host5", Optional.of("rack2")), createOffer(2, 2000, 2000, "agent6", "host6", Optional.of("rack3")))).join();
        Assertions.assertEquals(4, taskManager.getActiveTaskIds().size());
    } finally {
        configuration.setExpectedRacksCount(Optional.empty());
    }
}

Example 4 with SingularityKilledTaskIdRecord

use of com.hubspot.singularity.SingularityKilledTaskIdRecord in project Singularity by HubSpot.

the class SingularitySchedulerTest method testScaleDownTakesHighestInstances.

@Test
public void testScaleDownTakesHighestInstances() {
    initRequest();
    initFirstDeploy();
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(5)));
    resourceOffers();
    Assertions.assertEquals(5, taskManager.getActiveTaskIds().size());
    requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(2), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()), singularityUser);
    resourceOffers();
    cleaner.drainCleanupQueue();
    Assertions.assertEquals(3, taskManager.getKilledTaskIdRecords().size());
    for (SingularityKilledTaskIdRecord taskId : taskManager.getKilledTaskIdRecords()) {
        Assertions.assertTrue(taskId.getTaskId().getInstanceNo() > 2);
        scheduler.drainPendingQueue();
    }
}