Examples with ProvenanceEventRecordDTO com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO used on opensource projects

Example 6 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class FeedStatistics method addEvent.

public void addEvent(ProvenanceEventRecord event, Long eventId) {
    FeedEventStatistics.getInstance().calculateTimes(event, eventId);
    ProvenanceEventRecordDTO eventRecordDTO = null;
    String feedFlowFileId = FeedEventStatistics.getInstance().getFeedFlowFileId(event);
    boolean isStartingFeedFlow = ProvenanceEventUtil.isStartingFeedFlow(event);
    String batchKey = batchKey(event, feedFlowFileId, isStartingFeedFlow);
    // always track drop events if its on a tracked feed
    boolean isDropEvent = ProvenanceEventUtil.isEndingFlowFileEvent(event);
    if (isDropEvent && FeedEventStatistics.getInstance().beforeProcessingIsLastEventForTrackedFeed(event, eventId)) {
        batchKey += UUID.randomUUID().toString();
    }
    if (((!isStartingFeedFlow && FeedEventStatistics.getInstance().isTrackingDetails(event.getFlowFileUuid())) || (isStartingFeedFlow && lastRecords.size() <= limit)) && !lastRecords.containsKey(batchKey)) {
        // if we are tracking details send the event off for jms
        if (isStartingFeedFlow) {
            FeedEventStatistics.getInstance().setTrackingDetails(event);
        }
        eventRecordDTO = ProvenanceEventRecordConverter.convert(event);
        eventRecordDTO.setEventId(eventId);
        eventRecordDTO.setIsStartOfJob(ProvenanceEventUtil.isStartingFeedFlow(event));
        eventRecordDTO.setJobFlowFileId(feedFlowFileId);
        eventRecordDTO.setFirstEventProcessorId(feedProcessorId);
        eventRecordDTO.setStartTime(FeedEventStatistics.getInstance().getEventStartTime(eventId));
        eventRecordDTO.setEventDuration(FeedEventStatistics.getInstance().getEventDuration(eventId));
        if (ProvenanceEventUtil.isFlowFileQueueEmptied(event)) {
            // a Drop event component id will be the connection, not the processor id. we will set the name of the component
            eventRecordDTO.setComponentName("FlowFile Queue emptied");
            eventRecordDTO.setIsFailure(true);
        }
        if (ProvenanceEventUtil.isTerminatedByFailureRelationship(event)) {
            eventRecordDTO.setIsFailure(true);
        }
        lastRecords.put(batchKey, eventRecordDTO);
    } else {
        FeedEventStatistics.getInstance().skip(event, eventId);
    }
    FeedEventStatistics.getInstance().finishedEvent(event, eventId);
    boolean isEndingEvent = FeedEventStatistics.getInstance().isEndingFeedFlow(eventId);
    if (eventRecordDTO != null && isEndingEvent) {
        eventRecordDTO.setIsFinalJobEvent(isEndingEvent);
    }
    FeedProcessorStatisticsAggregator.getInstance().add(getStats(event), event, eventId);
    FeedEventStatistics.getInstance().cleanup(event, eventId);
}

Example 7 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class FeedStatisticsManager method gatherStatistics.

public void gatherStatistics() {
    lock.lock();
    List<ProvenanceEventRecordDTO> eventsToSend = null;
    Map<String, AggregatedFeedProcessorStatistics> statsToSend = null;
    try {
        // Gather Events and Stats to send Ops Manager
        // filter out the streaming feeds
        ensureStreamingFeedMetadata();
        eventsToSend = feedStatisticsMap.values().stream().flatMap(stats -> stats.getEventsToSend().stream().filter(event -> !FeedEventStatistics.getInstance().streamingFeedProcessorIdsList.contains(event.getFirstEventProcessorId()))).sorted(Comparator.comparing(ProvenanceEventRecordDTO::getEventTime).thenComparing(ProvenanceEventRecordDTO::getEventId)).collect(Collectors.toList());
        final String collectionId = UUID.randomUUID().toString();
        Map<String, Long> runningFlowsCount = new HashMap<>();
        for (FeedStatistics feedStatistics : feedStatisticsMap.values()) {
            if (feedStatistics.hasStats()) {
                if (statsToSend == null) {
                    statsToSend = new ConcurrentHashMap<>();
                }
                AggregatedFeedProcessorStatistics feedProcessorStatistics = statsToSend.computeIfAbsent(feedStatistics.getFeedProcessorId(), feedProcessorId -> new AggregatedFeedProcessorStatistics(feedStatistics.getFeedProcessorId(), collectionId, sendJmsTimeMillis));
                AggregatedProcessorStatistics processorStatistics = feedProcessorStatistics.getProcessorStats().computeIfAbsent(feedStatistics.getProcessorId(), processorId -> new AggregatedProcessorStatisticsV2(feedStatistics.getProcessorId(), null, collectionId));
                // accumulate the stats together into the processorStatistics object grouped by source connection id
                feedStatistics.getStats().stream().forEach(stats -> {
                    FeedProcessorStatisticsAggregator.getInstance().addStats1(processorStatistics.getStats(stats.getSourceConnectionIdentifier()), stats);
                });
            }
        }
        if ((eventsToSend != null && !eventsToSend.isEmpty()) || (statsToSend != null && !statsToSend.isEmpty())) {
            // send it off to jms on a different thread
            JmsSender jmsSender = new JmsSender(eventsToSend, statsToSend.values(), FeedEventStatistics.getInstance().getRunningFeedFlowsForFeed(statsToSend.keySet()));
            this.jmsService.submit(new JmsSenderConsumer(jmsSender));
        } else {
            // if we are empty but the runningFlows have changed, then send off as well
            if (FeedEventStatistics.getInstance().isFeedProcessorRunningFeedFlowsChanged()) {
                JmsSender jmsSender = new JmsSender(null, null, FeedEventStatistics.getInstance().getRunningFeedFlowsChanged());
                this.jmsService.submit(new JmsSenderConsumer(jmsSender));
            }
        }
    } finally {
        FeedEventStatistics.getInstance().markFeedProcessorRunningFeedFlowsUnchanged();
        feedStatisticsMap.values().stream().forEach(stats -> stats.clear());
        lock.unlock();
    }
}

Example 8 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class KyloJmsProvenanceEventService method sendEvents.

private void sendEvents(String jmsUrl, List<ProvenanceEventRecordDTO> events) throws Exception {
    ProvenanceEventRecordDTOHolder eventRecordDTOHolder = new ProvenanceEventRecordDTOHolder();
    List<ProvenanceEventRecordDTO> batchEvents = new ArrayList<>();
    for (ProvenanceEventRecordDTO event : events) {
        if (!event.isStream()) {
            batchEvents.add(event);
        }
    }
    eventRecordDTOHolder.setEvents(batchEvents);
    AggregatedFeedProcessorStatisticsHolder stats = GroupedStatsUtil.gatherStats(events);
    log.info("Sending {} events to JMS ", eventRecordDTOHolder);
    sendKyloBatchEventMessage(jmsUrl, eventRecordDTOHolder);
    sendKyloEventStatisticsMessage(jmsUrl, stats);
    log.info("Events successfully sent to JMS");
}

Example 9 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class TestProvenanceRest method testProvenanceRest.

// @Test
public void testProvenanceRest() {
    ProvenanceEventService restProvenanceEventService = new KyloRestProvenanceEventService();
    Map<String, String> params = new HashMap<>();
    params.put(KyloRestProvenanceEventService.USERNAME_CONFIG, "dladmin");
    params.put(KyloRestProvenanceEventService.PASSWORD_CONFIG, "thinkbig");
    params.put(KyloRestProvenanceEventService.HOST_CONFIG, "localhost");
    params.put(KyloRestProvenanceEventService.PORT_CONFIG, "8400");
    restProvenanceEventService.configure(params);
    String feedName = "provenance.provenance_test";
    String flowfileId = UUID.randomUUID().toString();
    DateTime startTime = DateTime.now().minusMinutes(1);
    Long start = startTime.getMillis();
    ProvenanceEventRecordDTO event1 = new ProvenanceEventDtoBuilder(feedName, flowfileId, "First Step").startingEvent(true).startTime(start).build();
    ProvenanceEventRecordDTO event2 = new ProvenanceEventDtoBuilder(feedName, flowfileId, "Second Step").startTime(startTime.plusSeconds(30).getMillis()).build();
    ProvenanceEventRecordDTO event3 = new ProvenanceEventDtoBuilder(feedName, flowfileId, "Final Step").endingEvent(true).build();
    List<ProvenanceEventRecordDTO> events = new ArrayList<>();
    events.add(event1);
    events.add(event2);
    events.add(event3);
    try {
        restProvenanceEventService.sendEvents(events);
    } catch (ProvenanceException e) {
        e.printStackTrace();
    }
}

Example 10 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class GroupedStatsUtil method gatherStats.

/**
 * Gather feed stats for a list of events
 */
public static AggregatedFeedProcessorStatisticsHolder gatherStats(final List<ProvenanceEventRecordDTO> events) {
    Map<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStatsByProcessor = new ConcurrentHashMap<>();
    // events.stream().forEach(e -> {
    for (ProvenanceEventRecordDTO e : events) {
        if (!feedStatsByProcessor.containsKey(e.getFeedName())) {
            feedStatsByProcessor.put(e.getFeedName(), new ConcurrentHashMap<GroupedStatsIdentity, List<GroupedStats>>());
        }
        // feedStatsByProcessor.putIfAbsent(e.getFeedName(), );
        Map<GroupedStatsIdentity, List<GroupedStats>> feedStats = feedStatsByProcessor.get(e.getFeedName());
        GroupedStatsIdentity identity = new GroupedStatsIdentity(e.getComponentId(), e.getComponentName());
        if (!feedStats.containsKey(identity)) {
            feedStats.put(identity, new ArrayList<GroupedStats>());
        }
        // feedStats.putIfAbsent(identity, new ArrayList<>());
        List<GroupedStats> feedProcessorStats = feedStats.get(identity);
        // Add the new stats
        GroupedStats statsV2 = GroupedStatsUtil.add(new GroupedStatsV2(), e);
        feedProcessorStats.add(statsV2);
    }
    // );
    List<AggregatedFeedProcessorStatistics> statsList = new ArrayList<>();
    for (Map.Entry<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStats : feedStatsByProcessor.entrySet()) {
        AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
        statsList.add(feedProcessorStatistics);
    }
    /* feedStatsByProcessor.entrySet().stream().forEach(feedStats -> {
            AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
            statsList.add(feedProcessorStatistics);
        });
        */
    AggregatedFeedProcessorStatisticsHolderV3 feedProcessorStatisticsHolderV3 = new AggregatedFeedProcessorStatisticsHolderV3();
    feedProcessorStatisticsHolderV3.setFeedStatistics(statsList);
    return feedProcessorStatisticsHolderV3;
}