Examples with AggregatedFeedProcessorStatisticsHolder com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolder used on opensource projects

Example 1 with AggregatedFeedProcessorStatisticsHolder

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolder in project kylo by Teradata.

the class KyloKafkaProvenanceEventService method sendEvents.

@Override
public void sendEvents(List<ProvenanceEventRecordDTO> events) throws ProvenanceException {
    try {
        List<Future<RecordMetadata>> resultFutures = new ArrayList<>();
        ProvenanceEventRecordDTOHolder eventRecordDTOHolder = new ProvenanceEventRecordDTOHolder();
        List<ProvenanceEventRecordDTO> batchEvents = new ArrayList<>();
        for (ProvenanceEventRecordDTO event : events) {
            if (!event.isStream()) {
                batchEvents.add(event);
            }
        }
        eventRecordDTOHolder.setEvents(batchEvents);
        byte[] data = SerializationUtils.serialize(eventRecordDTOHolder);
        ProducerRecord<byte[], byte[]> eventsMessage = new ProducerRecord<>(KYLO_BATCH_EVENT_TOPIC, data);
        log.info("Sending {} events to Kafka ", eventRecordDTOHolder);
        resultFutures.add(kafkaProducer.send(eventsMessage));
        AggregatedFeedProcessorStatisticsHolder stats = GroupedStatsUtil.gatherStats(events);
        data = SerializationUtils.serialize(stats);
        ProducerRecord<byte[], byte[]> statsMessage = new ProducerRecord<>(KYLO_EVENT_STATS_TOPIC, data);
        resultFutures.add(kafkaProducer.send(statsMessage));
        processAcks(resultFutures);
    } catch (Exception e) {
        throw new ProvenanceException(e);
    }
}

Example 2 with AggregatedFeedProcessorStatisticsHolder

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolder in project kylo by Teradata.

the class ProvenanceRestController method addProvenance.

@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("add custom provenance events to a job")
@ApiResponses(@ApiResponse(code = 200, message = "add custom provenance events to a job", response = String.class))
public Response addProvenance(ProvenanceEventRecordDTOHolder eventRecordDTOHolder) {
    ProvenanceEventRecordDTOHolder batchEventEntity = new ProvenanceEventRecordDTOHolder();
    List<ProvenanceEventRecordDTO> events = eventRecordDTOHolder.getEvents();
    List<ProvenanceEventRecordDTO> batchEvents = new ArrayList<>();
    for (ProvenanceEventRecordDTO event : events) {
        if (!event.isStream()) {
            batchEvents.add(event);
        }
    }
    // reassign the events
    batchEventEntity.setEvents(batchEvents);
    AggregatedFeedProcessorStatisticsHolder stats = GroupedStatsUtil.gatherStats(events);
    log.info("Processing {} batch  events", batchEventEntity);
    provenanceEventReceiver.receiveEvents(batchEventEntity);
    log.info("Processing {} stats ", stats);
    statsJmsReceiver.receiveTopic(stats);
    return Response.ok(new RestResponseStatus.ResponseStatusBuilder().message("Processed " + eventRecordDTOHolder).buildSuccess()).build();
}

Example 3 with AggregatedFeedProcessorStatisticsHolder

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolder in project kylo by Teradata.

the class KyloJmsProvenanceEventService method sendEvents.

private void sendEvents(String jmsUrl, List<ProvenanceEventRecordDTO> events) throws Exception {
    ProvenanceEventRecordDTOHolder eventRecordDTOHolder = new ProvenanceEventRecordDTOHolder();
    List<ProvenanceEventRecordDTO> batchEvents = new ArrayList<>();
    for (ProvenanceEventRecordDTO event : events) {
        if (!event.isStream()) {
            batchEvents.add(event);
        }
    }
    eventRecordDTOHolder.setEvents(batchEvents);
    AggregatedFeedProcessorStatisticsHolder stats = GroupedStatsUtil.gatherStats(events);
    log.info("Sending {} events to JMS ", eventRecordDTOHolder);
    sendKyloBatchEventMessage(jmsUrl, eventRecordDTOHolder);
    sendKyloEventStatisticsMessage(jmsUrl, stats);
    log.info("Events successfully sent to JMS");
}