Search in sources :

Example 1 with UpdateFailedDetails

use of com.sequenceiq.periscope.domain.UpdateFailedDetails in project cloudbreak by hortonworks.

the class UpdateFailedHandlerTest method getARunningCluster.

private Cluster getARunningCluster() {
    Cluster cluster = new Cluster();
    cluster.setId(AUTOSCALE_CLUSTER_ID);
    cluster.setStackCrn(CLOUDBREAK_STACK_CRN);
    cluster.setState(ClusterState.RUNNING);
    ClusterPertain clusterPertain = new ClusterPertain();
    cluster.setClusterPertain(clusterPertain);
    UpdateFailedDetails updateFailedDetails = new UpdateFailedDetails(Instant.now().minus(45, ChronoUnit.MINUTES).toEpochMilli(), 3L, true);
    cluster.setUpdateFailedDetails(updateFailedDetails);
    return cluster;
}
Also used : ClusterPertain(com.sequenceiq.periscope.domain.ClusterPertain) Cluster(com.sequenceiq.periscope.domain.Cluster) UpdateFailedDetails(com.sequenceiq.periscope.domain.UpdateFailedDetails)

Example 2 with UpdateFailedDetails

use of com.sequenceiq.periscope.domain.UpdateFailedDetails in project cloudbreak by hortonworks.

the class UpdateFailedHandler method onApplicationEvent.

@Override
public void onApplicationEvent(UpdateFailedEvent event) {
    long autoscaleClusterId = event.getClusterId();
    Cluster cluster = clusterService.findById(autoscaleClusterId);
    LoggingUtils.buildMdcContext(cluster);
    Integer failed = Optional.ofNullable(updateFailures.get(autoscaleClusterId)).map(failedCount -> failedCount + 1).orElse(1);
    if (failed < RETRY_THRESHOLD) {
        updateFailures.put(autoscaleClusterId, failed);
        LOGGER.debug("Increased failed count '{}' for cluster '{}'", failed, cluster.getStackCrn());
        if (event.getCausedBy() instanceof ForbiddenException && event.isWithMachineUser()) {
            Integer forbiddencount = getForbiddenFailureCount(cluster) + 1;
            UpdateFailedDetails updateFailedDetails = new UpdateFailedDetails(event.getLastExceptionTimestamp(), forbiddencount.longValue(), event.isWithMachineUser());
            clusterService.setUpdateFailedDetails(cluster.getId(), updateFailedDetails);
            LOGGER.debug("Increased forbidden count '{}' for cluster '{}' with failure details: {}, and machine user: {}", forbiddencount, cluster.getStackCrn(), updateFailedDetails, cluster.getMachineUserCrn());
        }
    } else {
        if (getForbiddenFailureCount(cluster) > 0) {
            LOGGER.info("Forbidden(403) failure(s) are present for cluster: {}, re-initialising polling machine user", cluster.getStackCrn());
            altusMachineUserService.initializeMachineUserForEnvironment(cluster);
            clusterService.setUpdateFailedDetails(cluster.getId(), null);
        }
        suspendCluster(cluster);
        updateFailures.remove(autoscaleClusterId);
        historyService.createEntry(ScalingStatus.TRIGGER_FAILED, messagesService.getMessage(MessageCode.AUTOSCALING_TRIGGER_FAILURE), cluster);
        LOGGER.debug("Suspended cluster monitoring for cluster '{}' due to failing update attempts", cluster.getStackCrn());
    }
}
Also used : Logger(org.slf4j.Logger) Cluster(com.sequenceiq.periscope.domain.Cluster) ForbiddenException(javax.ws.rs.ForbiddenException) MessageCode(com.sequenceiq.periscope.common.MessageCode) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) LoggerFactory(org.slf4j.LoggerFactory) ApplicationListener(org.springframework.context.ApplicationListener) UpdateFailedEvent(com.sequenceiq.periscope.monitor.event.UpdateFailedEvent) Inject(javax.inject.Inject) AltusMachineUserService(com.sequenceiq.periscope.service.AltusMachineUserService) CloudbreakMessagesService(com.sequenceiq.cloudbreak.message.CloudbreakMessagesService) Component(org.springframework.stereotype.Component) HistoryService(com.sequenceiq.periscope.service.HistoryService) LoggingUtils(com.sequenceiq.periscope.utils.LoggingUtils) Map(java.util.Map) ClusterState(com.sequenceiq.periscope.api.model.ClusterState) Optional(java.util.Optional) ScalingStatus(com.sequenceiq.periscope.api.model.ScalingStatus) UpdateFailedDetails(com.sequenceiq.periscope.domain.UpdateFailedDetails) ClusterService(com.sequenceiq.periscope.service.ClusterService) ForbiddenException(javax.ws.rs.ForbiddenException) Cluster(com.sequenceiq.periscope.domain.Cluster) UpdateFailedDetails(com.sequenceiq.periscope.domain.UpdateFailedDetails)

Aggregations

Cluster (com.sequenceiq.periscope.domain.Cluster)2 UpdateFailedDetails (com.sequenceiq.periscope.domain.UpdateFailedDetails)2 CloudbreakMessagesService (com.sequenceiq.cloudbreak.message.CloudbreakMessagesService)1 ClusterState (com.sequenceiq.periscope.api.model.ClusterState)1 ScalingStatus (com.sequenceiq.periscope.api.model.ScalingStatus)1 MessageCode (com.sequenceiq.periscope.common.MessageCode)1 ClusterPertain (com.sequenceiq.periscope.domain.ClusterPertain)1 UpdateFailedEvent (com.sequenceiq.periscope.monitor.event.UpdateFailedEvent)1 AltusMachineUserService (com.sequenceiq.periscope.service.AltusMachineUserService)1 ClusterService (com.sequenceiq.periscope.service.ClusterService)1 HistoryService (com.sequenceiq.periscope.service.HistoryService)1 LoggingUtils (com.sequenceiq.periscope.utils.LoggingUtils)1 Map (java.util.Map)1 Optional (java.util.Optional)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 Inject (javax.inject.Inject)1 ForbiddenException (javax.ws.rs.ForbiddenException)1 Logger (org.slf4j.Logger)1 LoggerFactory (org.slf4j.LoggerFactory)1 ApplicationListener (org.springframework.context.ApplicationListener)1