Search in sources :

Example 1 with BatchAndStreamingJobStatusCount

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchAndStreamingJobStatusCount in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method getBatchAndStreamingJobCounts.

/**
 * Get count of Jobs grouped by Status
 * Streaming Feeds are given a count of 1 if they are running, regardless of the number of active running flows
 */
public List<BatchAndStreamingJobStatusCount> getBatchAndStreamingJobCounts(String filter) {
    QJpaBatchJobExecution jobExecution = QJpaBatchJobExecution.jpaBatchJobExecution;
    QJpaBatchJobInstance jobInstance = QJpaBatchJobInstance.jpaBatchJobInstance;
    QJpaOpsManagerFeed feed = QJpaOpsManagerFeed.jpaOpsManagerFeed;
    QJpaNifiFeedStats feedStats = QJpaNifiFeedStats.jpaNifiFeedStats;
    BooleanBuilder whereBuilder = new BooleanBuilder();
    if (StringUtils.isNotBlank(filter)) {
        whereBuilder.and(GenericQueryDslFilter.buildFilter(jobExecution, filter));
    }
    Expression<JpaBatchAndStreamingJobStatusCounts> expr = Projections.bean(JpaBatchAndStreamingJobStatusCounts.class, JobStatusDslQueryExpressionBuilder.jobState().as("status"), feed.id.as("opsManagerFeedId"), feed.name.as("feedName"), feed.isStream.as("isStream"), feedStats.runningFeedFlows.as("runningFeedFlows"), jobExecution.jobExecutionId.count().as("count"), feedStats.lastActivityTimestamp.max().as("lastActivityTimestamp"));
    JPAQuery<?> query = factory.select(expr).from(feed).innerJoin(jobInstance).on(jobInstance.feed.id.eq(feed.id)).innerJoin(jobExecution).on(jobExecution.jobInstance.jobInstanceId.eq(jobInstance.jobInstanceId)).leftJoin(feedStats).on(feed.id.uuid.eq(feedStats.feedId.uuid)).where(whereBuilder).groupBy(jobExecution.status, feed.id, feed.name, feed.isStream, feedStats.runningFeedFlows);
    List<BatchAndStreamingJobStatusCount> stats = (List<BatchAndStreamingJobStatusCount>) query.fetch();
    return stats.stream().map(s -> {
        if (s.isStream() && (BatchJobExecution.RUNNING_DISPLAY_STATUS.equalsIgnoreCase(s.getStatus()) || BatchJobExecution.JobStatus.STARTING.name().equalsIgnoreCase(s.getStatus()) || BatchJobExecution.JobStatus.STARTED.name().equalsIgnoreCase(s.getStatus())) && s.getRunningFeedFlows() == 0L) {
            ((JpaBatchAndStreamingJobStatusCounts) s).setStatus(BatchJobExecution.JobStatus.STOPPED.name());
        }
        return s;
    }).collect(Collectors.toList());
// return stats;
}
Also used : AlertManager(com.thinkbiganalytics.alerts.spi.AlertManager) JobStatusCount(com.thinkbiganalytics.metadata.api.jobrepo.job.JobStatusCount) BatchRelatedFlowFile(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchRelatedFlowFile) ClusterServiceMessageReceiver(com.thinkbiganalytics.cluster.ClusterServiceMessageReceiver) JpaNifiEventJobExecution(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiEventJobExecution) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) Projections(com.querydsl.core.types.Projections) FeedAclIndexQueryAugmentor(com.thinkbiganalytics.metadata.jpa.feed.FeedAclIndexQueryAugmentor) StringUtils(org.apache.commons.lang3.StringUtils) GenericQueryDslFilter(com.thinkbiganalytics.metadata.jpa.support.GenericQueryDslFilter) SecurityRole(com.thinkbiganalytics.security.role.SecurityRole) BatchAndStreamingJobStatusCount(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchAndStreamingJobStatusCount) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) OpsManagerFeed(com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed) Pageable(org.springframework.data.domain.Pageable) BigInteger(java.math.BigInteger) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) SecurityContextHolder(org.springframework.security.core.context.SecurityContextHolder) NifiFeedStatisticsProvider(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedStatisticsProvider) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ClusterMessage(com.thinkbiganalytics.cluster.ClusterMessage) Page(org.springframework.data.domain.Page) Expression(com.querydsl.core.types.Expression) MetadataEventService(com.thinkbiganalytics.metadata.api.event.MetadataEventService) Collectors(java.util.stream.Collectors) JPAExpressions(com.querydsl.jpa.JPAExpressions) JpaOpsManagerFeed(com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed) JPAQuery(com.querydsl.jpa.impl.JPAQuery) BatchJobExecutionProvider(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecutionProvider) ReadablePeriod(org.joda.time.ReadablePeriod) List(java.util.List) ExecutionConstants(com.thinkbiganalytics.metadata.api.jobrepo.ExecutionConstants) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) PostConstruct(javax.annotation.PostConstruct) ClusterService(com.thinkbiganalytics.cluster.ClusterService) BatchJobExecution(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution) CommonFilterTranslations(com.thinkbiganalytics.metadata.jpa.support.CommonFilterTranslations) Predicate(com.querydsl.core.types.Predicate) SearchCriteria(com.thinkbiganalytics.metadata.api.SearchCriteria) Authentication(org.springframework.security.core.Authentication) UnsupportedEncodingException(java.io.UnsupportedEncodingException) FeedOperationBatchStatusChange(com.thinkbiganalytics.metadata.api.event.feed.FeedOperationBatchStatusChange) BatchStepExecutionProvider(com.thinkbiganalytics.metadata.api.jobrepo.step.BatchStepExecutionProvider) Stopwatch(com.google.common.base.Stopwatch) MessageDigest(java.security.MessageDigest) DateTimeUtil(com.thinkbiganalytics.DateTimeUtil) BooleanBuilder(com.querydsl.core.BooleanBuilder) QOpsManagerFeedId(com.thinkbiganalytics.metadata.jpa.feed.QOpsManagerFeedId) BooleanUtils(org.apache.commons.lang3.BooleanUtils) HashMap(java.util.HashMap) JPQLQuery(com.querydsl.jpa.JPQLQuery) DefaultAlertChangeEventContent(com.thinkbiganalytics.alerts.spi.DefaultAlertChangeEventContent) QueryDslFetchJoin(com.thinkbiganalytics.metadata.jpa.support.QueryDslFetchJoin) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) AlertProvider(com.thinkbiganalytics.alerts.api.AlertProvider) RoleSetExposingSecurityExpressionRoot(com.thinkbiganalytics.metadata.config.RoleSetExposingSecurityExpressionRoot) NifiRelatedRootFlowFilesRepository(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.NifiRelatedRootFlowFilesRepository) Alert(com.thinkbiganalytics.alerts.api.Alert) CheckDataStepConstants(com.thinkbiganalytics.jobrepo.common.constants.CheckDataStepConstants) Service(org.springframework.stereotype.Service) QueryDslPagingSupport(com.thinkbiganalytics.metadata.jpa.support.QueryDslPagingSupport) Named(javax.inject.Named) ProvenanceEventRecordDTO(com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO) FeedOperation(com.thinkbiganalytics.metadata.api.op.FeedOperation) Logger(org.slf4j.Logger) BatchJobInstance(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobInstance) FeedNameUtil(com.thinkbiganalytics.support.FeedNameUtil) OptimisticLockException(javax.persistence.OptimisticLockException) ConstructorExpression(com.querydsl.core.types.ConstructorExpression) DateTime(org.joda.time.DateTime) NifiFeedStats(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedStats) JPAQueryFactory(com.querydsl.jpa.impl.JPAQueryFactory) OperationalAlerts(com.thinkbiganalytics.metadata.api.alerts.OperationalAlerts) QJpaNifiFeedStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.QJpaNifiFeedStats) TimeUnit(java.util.concurrent.TimeUnit) JobStatusDslQueryExpressionBuilder(com.thinkbiganalytics.metadata.jpa.support.JobStatusDslQueryExpressionBuilder) QJpaOpsManagerFeed(com.thinkbiganalytics.metadata.jpa.feed.QJpaOpsManagerFeed) FeedConstants(com.thinkbiganalytics.jobrepo.common.constants.FeedConstants) Collections(java.util.Collections) OpsManagerFeedRepository(com.thinkbiganalytics.metadata.jpa.feed.OpsManagerFeedRepository) QJpaOpsManagerFeed(com.thinkbiganalytics.metadata.jpa.feed.QJpaOpsManagerFeed) BatchAndStreamingJobStatusCount(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchAndStreamingJobStatusCount) BooleanBuilder(com.querydsl.core.BooleanBuilder) List(java.util.List) ArrayList(java.util.ArrayList) QJpaNifiFeedStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.QJpaNifiFeedStats)

Aggregations

Stopwatch (com.google.common.base.Stopwatch)1 BooleanBuilder (com.querydsl.core.BooleanBuilder)1 ConstructorExpression (com.querydsl.core.types.ConstructorExpression)1 Expression (com.querydsl.core.types.Expression)1 Predicate (com.querydsl.core.types.Predicate)1 Projections (com.querydsl.core.types.Projections)1 JPAExpressions (com.querydsl.jpa.JPAExpressions)1 JPQLQuery (com.querydsl.jpa.JPQLQuery)1 JPAQuery (com.querydsl.jpa.impl.JPAQuery)1 JPAQueryFactory (com.querydsl.jpa.impl.JPAQueryFactory)1 DateTimeUtil (com.thinkbiganalytics.DateTimeUtil)1 Alert (com.thinkbiganalytics.alerts.api.Alert)1 AlertProvider (com.thinkbiganalytics.alerts.api.AlertProvider)1 AlertManager (com.thinkbiganalytics.alerts.spi.AlertManager)1 DefaultAlertChangeEventContent (com.thinkbiganalytics.alerts.spi.DefaultAlertChangeEventContent)1 ClusterMessage (com.thinkbiganalytics.cluster.ClusterMessage)1 ClusterService (com.thinkbiganalytics.cluster.ClusterService)1 ClusterServiceMessageReceiver (com.thinkbiganalytics.cluster.ClusterServiceMessageReceiver)1 CheckDataStepConstants (com.thinkbiganalytics.jobrepo.common.constants.CheckDataStepConstants)1 FeedConstants (com.thinkbiganalytics.jobrepo.common.constants.FeedConstants)1