Search in sources :

Example 1 with IdleRunAction

use of com.epam.pipeline.entity.monitoring.IdleRunAction in project cloud-pipeline by epam.

the class ResourceMonitoringManager method monitorResourceUsage.

public void monitorResourceUsage() {
    int idleTimeout = preferenceManager.getPreference(SYSTEM_MAX_IDLE_TIMEOUT_MINUTES);
    Map<String, PipelineRun> running = pipelineRunManager.loadRunningPipelineRuns().stream().filter(r -> DateUtils.nowUTC().isAfter(r.getProlongedAtTime().plus(idleTimeout, ChronoUnit.MINUTES))).collect(Collectors.toMap(PipelineRun::getPodId, r -> r));
    LOGGER.debug("Checking cpu stats for pipelines: " + running.keySet().stream().collect(Collectors.joining(", ")));
    LocalDateTime now = DateUtils.nowUTC();
    Map<String, Double> cpuMetrics = monitoringDao.loadCpuUsageRateMetrics(running.keySet(), now.minusMinutes(idleTimeout), now);
    LOGGER.debug("CPU Metrics received: " + cpuMetrics.entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).collect(Collectors.joining(", ")));
    double idleCpuLevel = preferenceManager.getPreference(SYSTEM_IDLE_CPU_THRESHOLD_PERCENT) / PERCENT;
    int actionTimeout = preferenceManager.getPreference(SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES);
    IdleRunAction action = IdleRunAction.valueOf(preferenceManager.getPreference(SystemPreferences.SYSTEM_IDLE_ACTION));
    List<PipelineRun> runsToUpdate = processRuns(running, cpuMetrics, idleCpuLevel, actionTimeout, action);
    pipelineRunManager.updatePipelineRunsLastNotification(runsToUpdate);
}
Also used : PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) SYSTEM_MAX_IDLE_TIMEOUT_MINUTES(com.epam.pipeline.manager.preference.SystemPreferences.SYSTEM_MAX_IDLE_TIMEOUT_MINUTES) SYSTEM_IDLE_CPU_THRESHOLD_PERCENT(com.epam.pipeline.manager.preference.SystemPreferences.SYSTEM_IDLE_CPU_THRESHOLD_PERCENT) Precision(org.apache.commons.math3.util.Precision) InstanceType(com.epam.pipeline.entity.cluster.InstanceType) MessageConstants(com.epam.pipeline.common.MessageConstants) PipelineRunManager(com.epam.pipeline.manager.pipeline.PipelineRunManager) LocalDateTime(java.time.LocalDateTime) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) HashMap(java.util.HashMap) Scheduled(org.springframework.scheduling.annotation.Scheduled) IdleRunAction(com.epam.pipeline.entity.monitoring.IdleRunAction) ArrayList(java.util.ArrayList) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) Pair(org.apache.commons.lang3.tuple.Pair) MessageHelper(com.epam.pipeline.common.MessageHelper) Service(org.springframework.stereotype.Service) Map(java.util.Map) NotificationType(com.epam.pipeline.entity.notification.NotificationSettings.NotificationType) Observable(io.reactivex.Observable) ConditionalOnProperty(org.springframework.boot.autoconfigure.condition.ConditionalOnProperty) DateUtils(com.epam.pipeline.entity.utils.DateUtils) Logger(org.slf4j.Logger) InstanceOfferManager(com.epam.pipeline.manager.cluster.InstanceOfferManager) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) AbstractSchedulingManager(com.epam.pipeline.manager.scheduling.AbstractSchedulingManager) TaskScheduler(org.springframework.scheduling.TaskScheduler) Collectors(java.util.stream.Collectors) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) List(java.util.List) ChronoUnit(java.time.temporal.ChronoUnit) SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES(com.epam.pipeline.manager.preference.SystemPreferences.SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES) PostConstruct(javax.annotation.PostConstruct) NotificationManager(com.epam.pipeline.manager.notification.NotificationManager) MonitoringESDao(com.epam.pipeline.dao.monitoring.MonitoringESDao) Collections(java.util.Collections) LocalDateTime(java.time.LocalDateTime) IdleRunAction(com.epam.pipeline.entity.monitoring.IdleRunAction)

Aggregations

MessageConstants (com.epam.pipeline.common.MessageConstants)1 MessageHelper (com.epam.pipeline.common.MessageHelper)1 MonitoringESDao (com.epam.pipeline.dao.monitoring.MonitoringESDao)1 InstanceType (com.epam.pipeline.entity.cluster.InstanceType)1 IdleRunAction (com.epam.pipeline.entity.monitoring.IdleRunAction)1 NotificationType (com.epam.pipeline.entity.notification.NotificationSettings.NotificationType)1 PipelineRun (com.epam.pipeline.entity.pipeline.PipelineRun)1 DateUtils (com.epam.pipeline.entity.utils.DateUtils)1 InstanceOfferManager (com.epam.pipeline.manager.cluster.InstanceOfferManager)1 NotificationManager (com.epam.pipeline.manager.notification.NotificationManager)1 PipelineRunManager (com.epam.pipeline.manager.pipeline.PipelineRunManager)1 PreferenceManager (com.epam.pipeline.manager.preference.PreferenceManager)1 SystemPreferences (com.epam.pipeline.manager.preference.SystemPreferences)1 SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES (com.epam.pipeline.manager.preference.SystemPreferences.SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES)1 SYSTEM_IDLE_CPU_THRESHOLD_PERCENT (com.epam.pipeline.manager.preference.SystemPreferences.SYSTEM_IDLE_CPU_THRESHOLD_PERCENT)1 SYSTEM_MAX_IDLE_TIMEOUT_MINUTES (com.epam.pipeline.manager.preference.SystemPreferences.SYSTEM_MAX_IDLE_TIMEOUT_MINUTES)1 AbstractSchedulingManager (com.epam.pipeline.manager.scheduling.AbstractSchedulingManager)1 Observable (io.reactivex.Observable)1 LocalDateTime (java.time.LocalDateTime)1 ChronoUnit (java.time.temporal.ChronoUnit)1