Examples with OpsManagerFeed com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed used on opensource projects

Example 16 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class FeedsRestController method getFeedHealthForFeed.

@GET
@Path("/health/{feedName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the detailed health status of the specified feed.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the health.", response = FeedStatus.class), @ApiResponse(code = 404, message = "The feed does not exist.", response = RestResponseStatus.class) })
public FeedStatus getFeedHealthForFeed(@Context HttpServletRequest request, @PathParam("feedName") String feedName) {
    this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);
    return metadataAccess.read(() -> {
        final FeedHealth feedHealth = getFeedHealthCounts(request, feedName);
        if (feedHealth != null) {
            return FeedModelTransform.feedStatus(Lists.newArrayList(feedHealth));
        }
        final OpsManagerFeed feed = opsFeedManagerFeedProvider.findByName(feedName);
        if (feed != null) {
            return FeedModelTransform.feedStatus(feed);
        } else {
            throw new NotFoundException();
        }
    });
}

Example 17 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class JobModelTransform method executedJob.

public static ExecutedJob executedJob(BatchJobExecution jobExecution) {
    DefaultExecutedJob job = (DefaultExecutedJob) executedJobSimple(jobExecution);
    Map<String, String> jobExecutionContext = jobExecution.getJobExecutionContextAsMap();
    if (jobExecutionContext != null) {
        job.setExecutionContext(new HashMap<>(jobExecutionContext));
    }
    OpsManagerFeed feed = jobExecution.getJobInstance().getFeed();
    if (feed != null) {
        job.setJobType(feed.getFeedType().name());
    }
    Map<String, String> jobParams = jobExecution.getJobParametersAsMap();
    if (jobParams != null) {
        job.setJobParameters(new HashMap<>(jobParams));
    }
    job.setExecutedSteps(executedSteps(jobExecution.getStepExecutions()));
    return job;
}

Example 18 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class OpsFeedManagerFeedProvider method save.

@Override
public OpsManagerFeed save(OpsManagerFeed.ID feedId, String systemName, boolean isStream, Long timeBetweenBatchJobs) {
    OpsManagerFeed feed = repository.findByIdWithoutAcl(feedId);
    if (feed == null) {
        if (ensureUniqueFeedName) {
            ensureAndRemoveDuplicateFeedsWithTheSameName(systemName, feedId);
        }
        feed = new JpaOpsManagerFeed();
        ((JpaOpsManagerFeed) feed).setName(systemName);
        ((JpaOpsManagerFeed) feed).setId((OpsManagerFeedId) feedId);
        ((JpaOpsManagerFeed) feed).setStream(isStream);
        ((JpaOpsManagerFeed) feed).setTimeBetweenBatchJobs(timeBetweenBatchJobs);
        NifiFeedStats stats = feedStatisticsProvider.findLatestStatsForFeedWithoutAccessControl(systemName);
        if (stats == null) {
            JpaNifiFeedStats newStats = new JpaNifiFeedStats(systemName, new JpaNifiFeedStats.OpsManagerFeedId(feedId.toString()));
            newStats.setRunningFeedFlows(0L);
            feedStatisticsProvider.saveLatestFeedStats(Lists.newArrayList(newStats));
        }
    } else {
        ((JpaOpsManagerFeed) feed).setStream(isStream);
        ((JpaOpsManagerFeed) feed).setTimeBetweenBatchJobs(timeBetweenBatchJobs);
    }
    feed = save(feed);
    return feed;
}

Example 19 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method abandonJob.

@Override
public BatchJobExecution abandonJob(Long executionId) {
    BatchJobExecution execution = findByJobExecutionId(executionId, false);
    if (execution != null && !execution.getStatus().equals(BatchJobExecution.JobStatus.ABANDONED)) {
        if (execution.getStartTime() == null) {
            execution.setStartTime(DateTimeUtil.getNowUTCTime());
        }
        execution.setStatus(BatchJobExecution.JobStatus.ABANDONED);
        if (execution.getEndTime() == null) {
            execution.setEndTime(DateTimeUtil.getNowUTCTime());
        }
        String abandonMessage = "Job manually abandoned @ " + DateTimeUtil.getNowFormattedWithTimeZone();
        String msg = execution.getExitMessage() != null ? execution.getExitMessage() + "\n" : "";
        msg += abandonMessage;
        execution.setExitMessage(msg);
        // also stop any running steps??
        // find the feed associated with the job
        OpsManagerFeed feed = execution.getJobInstance().getFeed();
        save(execution);
        jobExecutionChangedNotifier.notifyAbandoned(execution, feed, null);
        // clear the associated alert
        String alertId = execution.getJobExecutionContextAsMap().get(BatchJobExecutionProvider.KYLO_ALERT_ID_PROPERTY);
        if (StringUtils.isNotBlank(alertId)) {
            provider.respondTo(provider.resolve(alertId), (alert1, response) -> response.handle(abandonMessage));
        }
    }
    return execution;
}

Example 20 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class FeedHealthSummaryCache method fetchFeedSummary.

private List<? extends FeedSummary> fetchFeedSummary() {
    return metadataAccess.read(() -> {
        Stopwatch stopwatch = Stopwatch.createStarted();
        List<? extends FeedSummary> list = opsManagerFeedProvider.findFeedSummary();
        Map<String, FeedSummary> latestFeeds = new HashMap<>();
        // NOTE it could also populate the last job execution time since the above query gets a union of the running jobs along with the latest finished jobs by feed
        list.stream().sorted(byRunningStatus.thenComparing(byStartTime)).forEach(f -> {
            String feedId = f.getFeedId().toString();
            if (!latestFeeds.containsKey(feedId)) {
                latestFeeds.put(feedId, f);
            }
        });
        // add in initial feeds
        List<? extends OpsManagerFeed> allFeeds = opsManagerFeedProvider.findAllWithoutAcl();
        allFeeds.stream().filter(f -> !latestFeeds.containsKey(f.getId().toString())).forEach(f -> {
            JpaFeedSummary s = new JpaFeedSummary();
            s.setStream(f.isStream());
            s.setFeedId(UUID.fromString(f.getId().toString()));
            s.setFeedName(f.getName());
            s.setFeedType(f.getFeedType());
            s.setRunningCount(0L);
            s.setAbandonedCount(0L);
            s.setFailedCount(0L);
            s.setAllCount(0L);
            s.setCompletedCount(0L);
            s.setRunStatus(FeedSummary.RunStatus.INITIAL);
            s.setStatus(BatchJobExecution.JobStatus.UNKNOWN);
            latestFeeds.put(s.getFeedId().toString(), s);
        });
        stopwatch.stop();
        log.debug("Time to fetchAndDedupe FeedSummary: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
        return new ArrayList<>(latestFeeds.values());
    }, MetadataAccess.SERVICE);
}