Examples with SingularityDeployStatistics com.hubspot.singularity.SingularityDeployStatistics used on opensource projects

Example 1 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularitySchedulerTest method testScheduledNotification.

@Test
public void testScheduledNotification() {
    // run every hour
    schedule = "0 0 * * * ?";
    initScheduledRequest();
    initFirstDeploy();
    configuration.setWarnIfScheduledJobIsRunningForAtLeastMillis(Long.MAX_VALUE);
    configuration.setWarnIfScheduledJobIsRunningPastNextRunPct(200);
    final long now = System.currentTimeMillis();
    SingularityTask firstTask = launchTask(request, firstDeploy, now - TimeUnit.HOURS.toMillis(3), 1, TaskState.TASK_RUNNING);
    scheduledJobPoller.runActionOnPoll();
    Mockito.verify(mailer, Mockito.times(0)).sendTaskOverdueMail(ArgumentMatchers.<Optional<SingularityTask>>any(), ArgumentMatchers.<SingularityTaskId>any(), ArgumentMatchers.<SingularityRequest>any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong());
    configuration.setWarnIfScheduledJobIsRunningForAtLeastMillis(TimeUnit.HOURS.toMillis(1));
    scheduledJobPoller.runActionOnPoll();
    Mockito.verify(mailer, Mockito.times(1)).sendTaskOverdueMail(ArgumentMatchers.<Optional<SingularityTask>>any(), ArgumentMatchers.<SingularityTaskId>any(), ArgumentMatchers.<SingularityRequest>any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong());
    scheduledJobPoller.runActionOnPoll();
    Mockito.verify(mailer, Mockito.times(1)).sendTaskOverdueMail(ArgumentMatchers.<Optional<SingularityTask>>any(), ArgumentMatchers.<SingularityTaskId>any(), ArgumentMatchers.<SingularityRequest>any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong());
    statusUpdate(firstTask, TaskState.TASK_FINISHED);
    Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(requestId, firstDeployId);
    long oldAvg = deployStatistics.get().getAverageRuntimeMillis().get();
    Assert.assertTrue(deployStatistics.get().getNumTasks() == 1);
    Assert.assertTrue(deployStatistics.get().getAverageRuntimeMillis().get() > 1 && deployStatistics.get().getAverageRuntimeMillis().get() < TimeUnit.DAYS.toMillis(1));
    configuration.setWarnIfScheduledJobIsRunningForAtLeastMillis(1);
    SingularityTask secondTask = launchTask(request, firstDeploy, now - 500, 1, TaskState.TASK_RUNNING);
    scheduledJobPoller.runActionOnPoll();
    Mockito.verify(mailer, Mockito.times(1)).sendTaskOverdueMail(ArgumentMatchers.<Optional<SingularityTask>>any(), ArgumentMatchers.<SingularityTaskId>any(), ArgumentMatchers.<SingularityRequest>any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong());
    statusUpdate(secondTask, TaskState.TASK_FINISHED);
    deployStatistics = deployManager.getDeployStatistics(requestId, firstDeployId);
    Assert.assertTrue(deployStatistics.get().getNumTasks() == 2);
    Assert.assertTrue(deployStatistics.get().getAverageRuntimeMillis().get() > 1 && deployStatistics.get().getAverageRuntimeMillis().get() < oldAvg);
    saveRequest(request.toBuilder().setScheduledExpectedRuntimeMillis(Optional.of(1L)).build());
    SingularityTask thirdTask = launchTask(request, firstDeploy, now - 502, 1, TaskState.TASK_RUNNING);
    scheduledJobPoller.runActionOnPoll();
    Mockito.verify(mailer, Mockito.times(2)).sendTaskOverdueMail(ArgumentMatchers.<Optional<SingularityTask>>any(), ArgumentMatchers.<SingularityTaskId>any(), ArgumentMatchers.<SingularityRequest>any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong());
    taskManager.deleteTaskHistory(thirdTask.getTaskId());
    scheduledJobPoller.runActionOnPoll();
    Mockito.verify(mailer, Mockito.times(3)).sendTaskOverdueMail(ArgumentMatchers.<Optional<SingularityTask>>any(), ArgumentMatchers.<SingularityTaskId>any(), ArgumentMatchers.<SingularityRequest>any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong());
}

Example 2 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularityJobPoller method getExpectedRuntime.

private Optional<Long> getExpectedRuntime(SingularityRequest request, SingularityTaskId taskId) {
    if (request.getScheduledExpectedRuntimeMillis().isPresent()) {
        return request.getScheduledExpectedRuntimeMillis();
    } else {
        final Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
        if (deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent()) {
            return deployStatistics.get().getAverageRuntimeMillis();
        }
        String scheduleExpression = request.getScheduleTypeSafe() == ScheduleType.RFC5545 ? request.getSchedule().get() : request.getQuartzScheduleSafe();
        Date nextRunAtDate;
        try {
            if (request.getScheduleTypeSafe() == ScheduleType.RFC5545) {
                final RFC5545Schedule rfc5545Schedule = new RFC5545Schedule(scheduleExpression);
                nextRunAtDate = rfc5545Schedule.getNextValidTime();
            } else {
                final CronExpression cronExpression = new CronExpression(scheduleExpression);
                final Date startDate = new Date(taskId.getStartedAt());
                nextRunAtDate = cronExpression.getNextValidTimeAfter(startDate);
            }
            if (nextRunAtDate == null) {
                String msg = String.format("No next run date found for %s (%s)", taskId, scheduleExpression);
                LOG.warn(msg);
                exceptionNotifier.notify(msg, ImmutableMap.of("taskId", taskId.toString()));
                return Optional.absent();
            }
        } catch (ParseException | InvalidRecurrenceRuleException e) {
            LOG.warn("Unable to parse schedule of type {} for expression {} (taskId: {}, err: {})", request.getScheduleTypeSafe(), scheduleExpression, taskId, e);
            exceptionNotifier.notify(String.format("Unable to parse schedule (%s)", e.getMessage()), e, ImmutableMap.of("taskId", taskId.toString(), "scheduleExpression", scheduleExpression, "scheduleType", request.getScheduleTypeSafe().toString()));
            return Optional.absent();
        }
        return Optional.of(nextRunAtDate.getTime() - taskId.getStartedAt());
    }
}

Example 3 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularityScheduler method handlePendingRequestsForDeployKey.

private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) {
    final String requestId = deployKey.getRequestId();
    final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId);
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
    if (!isRequestActive(maybeRequest)) {
        LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest));
        obsoleteRequests.getAndIncrement();
        for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
            requestManager.deletePendingRequest(pendingRequest);
        }
        return;
    }
    SingularityRequestWithState request = maybeRequest.get();
    Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(requestId);
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey);
    List<SingularityPendingRequest> effectivePendingRequests = new ArrayList<>();
    // Things that are closest to now (ie smaller timestamps) should come first in the queue
    pendingRequestsForDeploy.sort(Comparator.comparingLong(SingularityPendingRequest::getTimestamp));
    int scheduledTasks = 0;
    for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
        final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request);
        if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) {
            LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest));
            obsoleteRequests.getAndIncrement();
            requestManager.deletePendingRequest(pendingRequest);
            continue;
        }
        int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy);
        if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) {
            LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest);
            heldForScheduledActiveTask.getAndIncrement();
            continue;
        }
        if (effectivePendingRequests.isEmpty()) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
            // If we are here, there is already an immediate of run of the scheduled task launched. Drop anything that would
            // leave a second instance of the request in the pending queue.
            requestManager.deletePendingRequest(pendingRequest);
        }
    // Any other subsequent requests are not honored until after the pending queue is cleared.
    }
    totalNewScheduledTasks.getAndAdd(scheduledTasks);
}

Example 4 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTask.

@Timed
public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
    if (wasActive) {
        taskManager.deleteActiveTask(taskId.getId());
    }
    if (!task.isPresent() || task.get().getTaskRequest().getRequest().isLoadBalanced()) {
        taskManager.createLBCleanupTask(taskId);
    }
    if (requestManager.isBouncing(taskId.getRequestId())) {
        List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(taskId.getRequestId());
        boolean foundBouncingTask = false;
        for (SingularityTaskId activeTaskId : activeTaskIds) {
            Optional<SingularityTaskHistoryUpdate> maybeCleaningUpdate = taskManager.getTaskHistoryUpdate(activeTaskId, ExtendedTaskState.TASK_CLEANING);
            if (maybeCleaningUpdate.isPresent()) {
                if (maybeCleaningUpdate.get().getStatusReason().or("").contains("BOUNCE")) {
                    // TaskCleanupType enum is included in status message
                    LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                    foundBouncingTask = true;
                    break;
                } else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) {
                    for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) {
                        if (previousUpdate.getStatusMessage().or("").contains("BOUNCE")) {
                            LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                            foundBouncingTask = true;
                            break;
                        }
                    }
                }
            }
        }
        if (!foundBouncingTask) {
            LOG.info("Bounce completed for request {}, no cleaning tasks due to bounce found", taskId.getRequestId());
            Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId());
            if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equals(taskId.getDeployId())) {
                requestManager.deleteExpiringObject(SingularityExpiringBounce.class, taskId.getRequestId());
            }
            requestManager.markBounceComplete(taskId.getRequestId());
        }
    }
    final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics(task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, status);
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.EXISTED) {
        return;
    }
    updateDeployStatistics(deployStatistics, taskId, task, timestamp, state, scheduleResult);
}

Example 5 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularityScheduler method updateDeployStatistics.

private void updateDeployStatistics(SingularityDeployStatistics deployStatistics, SingularityTaskId taskId, Optional<SingularityTask> task, long timestamp, ExtendedTaskState state, Optional<PendingType> scheduleResult) {
    SingularityDeployStatisticsBuilder bldr = deployStatistics.toBuilder();
    if (!state.isFailed()) {
        if (bldr.getAverageRuntimeMillis().isPresent()) {
            long newAvgRuntimeMillis = (bldr.getAverageRuntimeMillis().get() * bldr.getNumTasks() + (timestamp - taskId.getStartedAt())) / (bldr.getNumTasks() + 1);
            bldr.setAverageRuntimeMillis(Optional.of(newAvgRuntimeMillis));
        } else {
            bldr.setAverageRuntimeMillis(Optional.of(timestamp - taskId.getStartedAt()));
        }
    }
    if (task.isPresent()) {
        long dueTime = task.get().getTaskRequest().getPendingTask().getPendingTaskId().getNextRunAt();
        long startedAt = taskId.getStartedAt();
        if (bldr.getAverageSchedulingDelayMillis().isPresent()) {
            long newAverageSchedulingDelayMillis = (bldr.getAverageSchedulingDelayMillis().get() * bldr.getNumTasks() + (startedAt - dueTime)) / (bldr.getNumTasks() + 1);
            bldr.setAverageSchedulingDelayMillis(Optional.of(newAverageSchedulingDelayMillis));
        } else {
            bldr.setAverageSchedulingDelayMillis(Optional.of(startedAt - dueTime));
        }
        final SingularityDeployStatistics newStatistics = bldr.build();
        deployManager.saveDeployStatistics(newStatistics);
    }
    bldr.setNumTasks(bldr.getNumTasks() + 1);
    if (!bldr.getLastFinishAt().isPresent() || timestamp > bldr.getLastFinishAt().get()) {
        bldr.setLastFinishAt(Optional.of(timestamp));
        bldr.setLastTaskState(Optional.of(state));
    }
    final ListMultimap<Integer, Long> instanceSequentialFailureTimestamps = bldr.getInstanceSequentialFailureTimestamps();
    final List<Long> sequentialFailureTimestamps = instanceSequentialFailureTimestamps.get(taskId.getInstanceNo());
    if (!state.isSuccess()) {
        if (SingularityTaskHistoryUpdate.getUpdate(taskManager.getTaskHistoryUpdates(taskId), ExtendedTaskState.TASK_CLEANING).isPresent()) {
            LOG.debug("{} failed with {} after cleaning - ignoring it for cooldown", taskId, state);
        } else {
            if (sequentialFailureTimestamps.size() < configuration.getCooldownAfterFailures()) {
                sequentialFailureTimestamps.add(timestamp);
            } else if (timestamp > sequentialFailureTimestamps.get(0)) {
                sequentialFailureTimestamps.set(0, timestamp);
            }
            bldr.setNumFailures(bldr.getNumFailures() + 1);
            Collections.sort(sequentialFailureTimestamps);
        }
    } else {
        bldr.setNumSuccess(bldr.getNumSuccess() + 1);
        sequentialFailureTimestamps.clear();
    }
    if (scheduleResult.isPresent() && scheduleResult.get() == PendingType.RETRY) {
        bldr.setNumSequentialRetries(bldr.getNumSequentialRetries() + 1);
    } else {
        bldr.setNumSequentialRetries(0);
    }
    final SingularityDeployStatistics newStatistics = bldr.build();
    LOG.trace("Saving new deploy statistics {}", newStatistics);
    deployManager.saveDeployStatistics(newStatistics);
}