Examples with EventHubConfig org.apache.samza.system.eventhub.EventHubConfig used on opensource projects

Example 1 with EventHubConfig

use of org.apache.samza.system.eventhub.EventHubConfig in project samza by apache.

the class TestEventHubSystemProducer method testSendingLargeMessage.

@Test
public void testSendingLargeMessage() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testLMStream";
    int numEvents = 10;
    int partitionId0 = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
    outgoingMessagesP0.add("1234567890123456789012345678901234567890");
    outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "30");
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size() + 1);
}

Example 2 with EventHubConfig

use of org.apache.samza.system.eventhub.EventHubConfig in project samza by apache.

the class TestEventHubSystemProducer method testSkipLargeMessageCheck.

@Test
public void testSkipLargeMessageCheck() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testLMStream";
    int numEvents = 10;
    int partitionId0 = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
    outgoingMessagesP0.add("1234567890123456789012345678901234567890");
    outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "-1");
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size());
}

Example 3 with EventHubConfig

use of org.apache.samza.system.eventhub.EventHubConfig in project samza by apache.

the class TestEventHubSystemProducer method testSendingToSpecificPartitions.

private void testSendingToSpecificPartitions(boolean perPartitionConnection) throws Exception {
    String systemName = "eventhubs";
    String streamName = "testStream";
    int numEvents = 10;
    int partitionId0 = 0;
    int partitionId1 = 1;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents);
    List<String> outgoingMessagesP1 = generateMessages(numEvents);
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    configMap.put(String.format(EventHubConfig.CONFIG_PER_PARTITION_CONNECTION, systemName), String.valueOf(perPartitionConnection));
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    outgoingMessagesP1.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    List<String> receivedData1 = factory.getSentData(systemName, streamName, partitionId1).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertTrue(outgoingMessagesP0.equals(receivedData0));
    Assert.assertTrue(outgoingMessagesP1.equals(receivedData1));
    if (perPartitionConnection) {
        Assert.assertNotEquals("perPartitionConnection=true; partitions should not share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
    } else {
        Assert.assertEquals("perPartitionConnection=false; partitions should share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
    }
}

Example 4 with EventHubConfig

use of org.apache.samza.system.eventhub.EventHubConfig in project samza by apache.

the class TestEventHubSystemAdmin method testStartpointResolverShouldResolveTheStartpointSpecificToCorrectOffset.

@Test
public void testStartpointResolverShouldResolveTheStartpointSpecificToCorrectOffset() {
    EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
    EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
    EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
    Assert.assertEquals("100", resolver.visit(systemStreamPartition, new StartpointSpecific("100")));
}

Example 5 with EventHubConfig

use of org.apache.samza.system.eventhub.EventHubConfig in project samza by apache.

the class TestEventHubSystemAdmin method testStartpointResolverShouldResolveTheStartpointTimestampToCorrectOffset.

@Test
public void testStartpointResolverShouldResolveTheStartpointTimestampToCorrectOffset() throws EventHubException {
    // Initialize variables required for testing.
    EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
    EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
    String mockedOffsetToReturn = "100";
    // Setup the mock variables.
    EventHubClientManager mockEventHubClientManager = Mockito.mock(EventHubClientManager.class);
    EventHubClient mockEventHubClient = Mockito.mock(EventHubClient.class);
    PartitionReceiver mockPartitionReceiver = Mockito.mock(PartitionReceiver.class);
    EventData mockEventData = Mockito.mock(EventData.class);
    EventData.SystemProperties mockSystemProperties = Mockito.mock(EventData.SystemProperties.class);
    // Configure the mock variables to return the appropriate values.
    Mockito.when(mockEventHubSystemAdmin.getOrCreateStreamEventHubClient("test-stream")).thenReturn(mockEventHubClientManager);
    Mockito.when(mockEventHubClientManager.getEventHubClient()).thenReturn(mockEventHubClient);
    Mockito.when(mockEventHubClient.createReceiverSync(Mockito.anyString(), Mockito.anyString(), Mockito.any())).thenReturn(mockPartitionReceiver);
    Mockito.when(mockPartitionReceiver.receiveSync(1)).thenReturn(Arrays.asList(mockEventData));
    Mockito.when(mockEventData.getSystemProperties()).thenReturn(mockSystemProperties);
    Mockito.when(mockSystemProperties.getOffset()).thenReturn(mockedOffsetToReturn);
    // Test the Offset resolver.
    EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
    String resolvedOffset = resolver.visit(systemStreamPartition, new StartpointTimestamp(100L));
    Assert.assertEquals(mockedOffsetToReturn, resolvedOffset);
}