use of org.apache.samza.system.eventhub.TestMetricsRegistry in project samza by apache.
the class TestEventHubSystemProducer method testSendingLargeMessage.
@Test
public void testSendingLargeMessage() throws Exception {
String systemName = "eventhubs";
String streamName = "testLMStream";
int numEvents = 10;
int partitionId0 = 0;
TestMetricsRegistry testMetrics = new TestMetricsRegistry();
Map<String, Interceptor> interceptor = new HashMap<>();
interceptor.put(streamName, new PassThroughInterceptor());
List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
outgoingMessagesP0.add("1234567890123456789012345678901234567890");
outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
// Set configs
Map<String, String> configMap = new HashMap<>();
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "30");
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
MapConfig config = new MapConfig(configMap);
MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
SystemStream systemStream = new SystemStream(systemName, streamName);
producer.register(SOURCE);
producer.start();
outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
// Retrieve sent data
List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size() + 1);
}
use of org.apache.samza.system.eventhub.TestMetricsRegistry in project samza by apache.
the class TestEventHubSystemProducer method testSkipLargeMessageCheck.
@Test
public void testSkipLargeMessageCheck() throws Exception {
String systemName = "eventhubs";
String streamName = "testLMStream";
int numEvents = 10;
int partitionId0 = 0;
TestMetricsRegistry testMetrics = new TestMetricsRegistry();
Map<String, Interceptor> interceptor = new HashMap<>();
interceptor.put(streamName, new PassThroughInterceptor());
List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
outgoingMessagesP0.add("1234567890123456789012345678901234567890");
outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
// Set configs
Map<String, String> configMap = new HashMap<>();
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "-1");
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
MapConfig config = new MapConfig(configMap);
MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
SystemStream systemStream = new SystemStream(systemName, streamName);
producer.register(SOURCE);
producer.start();
outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
// Retrieve sent data
List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size());
}
use of org.apache.samza.system.eventhub.TestMetricsRegistry in project samza by apache.
the class TestEventHubSystemProducer method testSendingToSpecificPartitions.
private void testSendingToSpecificPartitions(boolean perPartitionConnection) throws Exception {
String systemName = "eventhubs";
String streamName = "testStream";
int numEvents = 10;
int partitionId0 = 0;
int partitionId1 = 1;
TestMetricsRegistry testMetrics = new TestMetricsRegistry();
Map<String, Interceptor> interceptor = new HashMap<>();
interceptor.put(streamName, new PassThroughInterceptor());
List<String> outgoingMessagesP0 = generateMessages(numEvents);
List<String> outgoingMessagesP1 = generateMessages(numEvents);
// Set configs
Map<String, String> configMap = new HashMap<>();
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
configMap.put(String.format(EventHubConfig.CONFIG_PER_PARTITION_CONNECTION, systemName), String.valueOf(perPartitionConnection));
MapConfig config = new MapConfig(configMap);
MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
SystemStream systemStream = new SystemStream(systemName, streamName);
producer.register(SOURCE);
producer.start();
outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
outgoingMessagesP1.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
// Retrieve sent data
List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
List<String> receivedData1 = factory.getSentData(systemName, streamName, partitionId1).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
Assert.assertTrue(outgoingMessagesP0.equals(receivedData0));
Assert.assertTrue(outgoingMessagesP1.equals(receivedData1));
if (perPartitionConnection) {
Assert.assertNotEquals("perPartitionConnection=true; partitions should not share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
} else {
Assert.assertEquals("perPartitionConnection=false; partitions should share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
}
}
use of org.apache.samza.system.eventhub.TestMetricsRegistry in project samza by apache.
the class ITestEventHubSystemConsumer method testSinglePartitionConsumptionHappyPath.
@Test
public void testSinglePartitionConsumptionHappyPath() throws Exception {
int partitionId = 0;
TestMetricsRegistry testMetrics = new TestMetricsRegistry();
SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(partitionId));
Config eventHubConfig = createEventHubConfig();
EventHubSystemFactory factory = new EventHubSystemFactory();
SystemConsumer consumer = factory.getConsumer(SYSTEM_NAME, eventHubConfig, testMetrics);
consumer.register(ssp, EventHubSystemConsumer.START_OF_STREAM);
consumer.start();
int numEvents = 0;
int numRetries = 20;
while (numRetries-- > 0) {
List<IncomingMessageEnvelope> result = consumer.poll(Collections.singleton(ssp), 2000).get(ssp);
numEvents = result == null ? 0 : result.size();
if (numEvents > 0) {
EventHubIncomingMessageEnvelope eventData = (EventHubIncomingMessageEnvelope) result.get(0);
System.out.println("System properties: " + eventData.getEventData().getSystemProperties());
System.out.println("Key: " + new String((byte[]) eventData.getKey()));
System.out.println("Message: " + new String((byte[]) eventData.getMessage()));
}
System.out.println("Retries left: " + numRetries);
}
Assert.assertTrue(numEvents > 0);
}
use of org.apache.samza.system.eventhub.TestMetricsRegistry in project samza by apache.
the class TestEventHubSystemProducer method testSendingToSpecificPartitionsWithInterceptor.
@Test
public void testSendingToSpecificPartitionsWithInterceptor() throws Exception {
String systemName = "eventhubs";
String streamName = "testStream";
int numEvents = 10;
int partitionId0 = 0;
int partitionId1 = 1;
Interceptor interceptor = new SwapFirstLastByteInterceptor();
TestMetricsRegistry testMetrics = new TestMetricsRegistry();
Map<String, Interceptor> interceptors = new HashMap<>();
interceptors.put(streamName, interceptor);
List<String> outgoingMessagesP0 = generateMessages(numEvents);
List<String> outgoingMessagesP1 = generateMessages(numEvents);
// Set configs
Map<String, String> configMap = new HashMap<>();
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
MapConfig config = new MapConfig(configMap);
MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptors, testMetrics);
SystemStream systemStream = new SystemStream(systemName, streamName);
producer.register(SOURCE);
producer.start();
outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
outgoingMessagesP1.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
// Retrieve sent data
List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
List<String> receivedData1 = factory.getSentData(systemName, streamName, partitionId1).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
List<String> expectedP0 = outgoingMessagesP0.stream().map(message -> new String(interceptor.intercept(message.getBytes()))).collect(Collectors.toList());
List<String> expectedP1 = outgoingMessagesP1.stream().map(message -> new String(interceptor.intercept(message.getBytes()))).collect(Collectors.toList());
Assert.assertTrue(expectedP0.equals(receivedData0));
Assert.assertTrue(expectedP1.equals(receivedData1));
}
Aggregations