Examples with SingularityDisasterType com.hubspot.singularity.SingularityDisasterType used on opensource projects

Example 1 with SingularityDisasterType

use of com.hubspot.singularity.SingularityDisasterType in project Singularity by HubSpot.

the class DisasterManager method addDisabledActionsForDisasters.

public void addDisabledActionsForDisasters(List<SingularityDisasterType> newActiveDisasters) {
    boolean automaticallyClearable = true;
    for (SingularityDisasterType disasterType : newActiveDisasters) {
        if (!disasterType.isAutomaticallyClearable()) {
            automaticallyClearable = false;
            break;
        }
    }
    String message = String.format("Active disasters detected: (%s)%s", newActiveDisasters, automaticallyClearable ? "" : ", action must be re-enabled by an admin user");
    Optional<Long> expiresAt = Optional.absent();
    if (automaticallyClearable) {
        expiresAt = Optional.of(System.currentTimeMillis() + configuration.getDisasterDetection().getDefaultDisabledActionExpiration());
    }
    for (SingularityAction action : configuration.getDisasterDetection().getDisableActionsOnDisaster()) {
        disable(action, Optional.of(message), Optional.<SingularityUser>absent(), automaticallyClearable, expiresAt);
    }
}

Example 2 with SingularityDisasterType

use of com.hubspot.singularity.SingularityDisasterType in project Singularity by HubSpot.

the class SingularityDisasterDetectionPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    LOG.trace("Starting disaster detection");
    clearExpiredDisabledActions();
    List<SingularityDisasterType> previouslyActiveDisasters = disasterManager.getActiveDisasters();
    List<SingularityDisasterDataPoint> dataPoints = disasterManager.getDisasterStats().getDataPoints();
    SingularityDisasterDataPoint newStats = collectDisasterStats();
    dataPoints.add(0, newStats);
    if (dataPoints.size() > disasterConfiguration.getStatsHistorySize()) {
        dataPoints.remove(dataPoints.size() - 1);
    }
    LOG.debug("Collected new disaster detection dataPoints: {}", newStats);
    List<SingularityDisasterType> newActiveDisasters = checkDataPoints(dataPoints);
    if (!newActiveDisasters.isEmpty()) {
        LOG.warn("Detected new active disasters: {}", newActiveDisasters);
    }
    disasterManager.updateActiveDisasters(previouslyActiveDisasters, newActiveDisasters);
    disasterManager.saveDisasterStats(new SingularityDisasterDataPoints(dataPoints));
    if (!newActiveDisasters.isEmpty()) {
        if (!disasterManager.isAutomatedDisabledActionsDisabled()) {
            disasterManager.addDisabledActionsForDisasters(newActiveDisasters);
        }
        if (!previouslyActiveDisasters.containsAll(newActiveDisasters)) {
            queueDisasterEmail(dataPoints, newActiveDisasters);
        }
    } else {
        disasterManager.clearSystemGeneratedDisabledActions();
    }
}