Examples with EventData com.microsoft.azure.eventhubs.EventData used on opensource projects

Example 1 with EventData

use of com.microsoft.azure.eventhubs.EventData in project nifi by apache.

the class GetAzureEventHub method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final BlockingQueue<String> partitionIds = this.partitionNames;
    final String partitionId = partitionIds.poll();
    if (partitionId == null) {
        getLogger().debug("No partitions available");
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        final Iterable<EventData> receivedEvents = receiveEvents(context, partitionId);
        if (receivedEvents == null) {
            return;
        }
        for (final EventData eventData : receivedEvents) {
            if (null != eventData) {
                final Map<String, String> attributes = new HashMap<>();
                FlowFile flowFile = session.create();
                final EventData.SystemProperties systemProperties = eventData.getSystemProperties();
                if (null != systemProperties) {
                    attributes.put("eventhub.enqueued.timestamp", String.valueOf(systemProperties.getEnqueuedTime()));
                    attributes.put("eventhub.offset", systemProperties.getOffset());
                    attributes.put("eventhub.sequence", String.valueOf(systemProperties.getSequenceNumber()));
                }
                attributes.put("eventhub.name", context.getProperty(EVENT_HUB_NAME).getValue());
                attributes.put("eventhub.partition", partitionId);
                flowFile = session.putAllAttributes(flowFile, attributes);
                flowFile = session.write(flowFile, out -> {
                    out.write(eventData.getBytes());
                });
                session.transfer(flowFile, REL_SUCCESS);
                final String namespace = context.getProperty(NAMESPACE).getValue();
                final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue();
                final String consumerGroup = context.getProperty(CONSUMER_GROUP).getValue();
                final String serviceBusEndPoint = context.getProperty(SERVICE_BUS_ENDPOINT).getValue();
                final String transitUri = "amqps://" + namespace + serviceBusEndPoint + "/" + eventHubName + "/ConsumerGroups/" + consumerGroup + "/Partitions/" + partitionId;
                session.getProvenanceReporter().receive(flowFile, transitUri, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
            }
        }
    } finally {
        partitionIds.offer(partitionId);
    }
}

Example 2 with EventData

use of com.microsoft.azure.eventhubs.EventData in project nifi by apache.

the class TestConsumeAzureEventHub method testCheckpointFailure.

@Test
public void testCheckpointFailure() throws Exception {
    final Iterable<EventData> eventDataList = Arrays.asList(new EventData("one".getBytes(StandardCharsets.UTF_8)), new EventData("two".getBytes(StandardCharsets.UTF_8)));
    doThrow(new RuntimeException("Failed to create a checkpoint.")).when(partitionContext).checkpoint();
    eventProcessor.onEvents(partitionContext, eventDataList);
    // Even if it fails to create a checkpoint, these FlowFiles are already committed.
    processSession.assertCommitted();
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_SUCCESS);
    assertEquals(2, flowFiles.size());
    final MockFlowFile msg1 = flowFiles.get(0);
    msg1.assertContentEquals("one");
    final MockFlowFile msg2 = flowFiles.get(1);
    msg2.assertContentEquals("two");
    final List<ProvenanceEventRecord> provenanceEvents = sharedState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
}

Example 3 with EventData

use of com.microsoft.azure.eventhubs.EventData in project nifi by apache.

the class TestConsumeAzureEventHub method testReceiveRecordReaderFailure.

@Test
public void testReceiveRecordReaderFailure() throws Exception {
    final List<EventData> eventDataList = Arrays.asList(new EventData("one".getBytes(StandardCharsets.UTF_8)), new EventData("two".getBytes(StandardCharsets.UTF_8)), new EventData("three".getBytes(StandardCharsets.UTF_8)), new EventData("four".getBytes(StandardCharsets.UTF_8)));
    setupRecordReader(eventDataList, 2, null);
    setupRecordWriter();
    eventProcessor.onEvents(partitionContext, eventDataList);
    processSession.assertCommitted();
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_SUCCESS);
    assertEquals(1, flowFiles.size());
    final MockFlowFile ff1 = flowFiles.get(0);
    ff1.assertContentEquals("onetwofour");
    ff1.assertAttributeEquals("eventhub.name", "eventhub-name");
    ff1.assertAttributeEquals("eventhub.partition", "partition-id");
    final List<MockFlowFile> failedFFs = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_PARSE_FAILURE);
    assertEquals(1, failedFFs.size());
    final MockFlowFile failed1 = failedFFs.get(0);
    failed1.assertContentEquals("three");
    failed1.assertAttributeEquals("eventhub.name", "eventhub-name");
    failed1.assertAttributeEquals("eventhub.partition", "partition-id");
    final List<ProvenanceEventRecord> provenanceEvents = sharedState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
    final ProvenanceEventRecord provenanceEvent1 = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent1.getEventType());
    assertEquals("amqps://namespace.servicebus.windows.net/" + "eventhub-name/ConsumerGroups/consumer-group/Partitions/partition-id", provenanceEvent1.getTransitUri());
    final ProvenanceEventRecord provenanceEvent2 = provenanceEvents.get(1);
    assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent2.getEventType());
    assertEquals("amqps://namespace.servicebus.windows.net/" + "eventhub-name/ConsumerGroups/consumer-group/Partitions/partition-id", provenanceEvent2.getTransitUri());
}

Example 4 with EventData

use of com.microsoft.azure.eventhubs.EventData in project nifi by apache.

the class TestConsumeAzureEventHub method testReceiveTwo.

@Test
public void testReceiveTwo() throws Exception {
    final Iterable<EventData> eventDataList = Arrays.asList(new EventData("one".getBytes(StandardCharsets.UTF_8)), new EventData("two".getBytes(StandardCharsets.UTF_8)));
    eventProcessor.onEvents(partitionContext, eventDataList);
    processSession.assertCommitted();
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_SUCCESS);
    assertEquals(2, flowFiles.size());
    final MockFlowFile msg1 = flowFiles.get(0);
    msg1.assertContentEquals("one");
    final MockFlowFile msg2 = flowFiles.get(1);
    msg2.assertContentEquals("two");
    final List<ProvenanceEventRecord> provenanceEvents = sharedState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
}

Example 5 with EventData

use of com.microsoft.azure.eventhubs.EventData in project zipkin-azure by openzipkin.

the class ZipkinEventProcessorTest method parallelCheckpoint.

/**
 * This shows that checkpointing is consistent when callbacks are on different threads.
 */
@Test
public void parallelCheckpoint() throws Exception {
    int spansPerEvent = 3;
    // We checkpoint at or over the checkpoint batch size. By default, our batch size is
    // 10, so if we have 3 spans per event, we checkpoint on the 3rd event (span count 12 not 10).
    int eventsPerCheckpoint = processor.checkpointBatchSize / spansPerEvent;
    if (processor.checkpointBatchSize % spansPerEvent > 0)
        eventsPerCheckpoint++;
    // make a lot of events to ensure concurrency works.
    int eventCount = 1000;
    final ConcurrentLinkedQueue<EventData> events = new ConcurrentLinkedQueue<>();
    for (int i = 0; i < eventCount; i++) {
        events.add(jsonMessageWithThreeSpans(Integer.toHexString(i + 1), 1 + i));
    }
    // We currently don't know if onEvents is always called from the same thread or not.
    // 
    // To test logic is consistent, we fire up 10 threads who will pull events of the queue and
    // invoke onEvents with that event. This will happen concurrently and out-of-order.
    // If we don't end up with an exact number of checkpoints, we might have a concurrency bug.
    CountDownLatch latch = new CountDownLatch(events.size());
    int threadCount = 10;
    ExecutorService exec = Executors.newFixedThreadPool(threadCount);
    for (int i = 0; i < threadCount; i++) {
        exec.execute(() -> {
            EventData event;
            while ((event = events.poll()) != null) {
                try {
                    processor.onEvents(context, asList(event));
                } catch (Exception e) {
                    e.printStackTrace();
                }
                latch.countDown();
            }
        });
    }
    // latch.await();
    exec.shutdown();
    exec.awaitTermination(1, TimeUnit.SECONDS);
    assertThat(processor.countSinceCheckpoint).isZero();
    assertThat(checkpointEvents).hasSize(eventCount / eventsPerCheckpoint);
}