Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 11 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreation.

@Test
public void testStreamCreation() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}

Example 12 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestEventHubSystemProducer method testSendingLargeMessage.

@Test
public void testSendingLargeMessage() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testLMStream";
    int numEvents = 10;
    int partitionId0 = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
    outgoingMessagesP0.add("1234567890123456789012345678901234567890");
    outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "30");
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size() + 1);
}

Example 13 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestEventHubSystemProducer method testSkipLargeMessageCheck.

@Test
public void testSkipLargeMessageCheck() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testLMStream";
    int numEvents = 10;
    int partitionId0 = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
    outgoingMessagesP0.add("1234567890123456789012345678901234567890");
    outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "-1");
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size());
}

Example 14 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestEventHubSystemProducer method testSendingToSpecificPartitions.

private void testSendingToSpecificPartitions(boolean perPartitionConnection) throws Exception {
    String systemName = "eventhubs";
    String streamName = "testStream";
    int numEvents = 10;
    int partitionId0 = 0;
    int partitionId1 = 1;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents);
    List<String> outgoingMessagesP1 = generateMessages(numEvents);
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    configMap.put(String.format(EventHubConfig.CONFIG_PER_PARTITION_CONNECTION, systemName), String.valueOf(perPartitionConnection));
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    outgoingMessagesP1.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    List<String> receivedData1 = factory.getSentData(systemName, streamName, partitionId1).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertTrue(outgoingMessagesP0.equals(receivedData0));
    Assert.assertTrue(outgoingMessagesP1.equals(receivedData1));
    if (perPartitionConnection) {
        Assert.assertNotEquals("perPartitionConnection=true; partitions should not share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
    } else {
        Assert.assertEquals("perPartitionConnection=false; partitions should share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
    }
}

Example 15 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestEventHubSystemConsumer method testMultiStreamsConsumptionHappyPath.

@Test
public void testMultiStreamsConsumptionHappyPath() throws Exception {
    String systemName = "eventhubs";
    String streamName1 = "testStream1";
    String streamName2 = "testStream2";
    // needs to be less than BLOCKING_QUEUE_SIZE
    int numEvents = 10;
    int partitionId = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<SystemStreamPartition, List<EventData>> eventData = new HashMap<>();
    SystemStreamPartition ssp1 = new SystemStreamPartition(systemName, streamName1, new Partition(partitionId));
    SystemStreamPartition ssp2 = new SystemStreamPartition(systemName, streamName2, new Partition(partitionId));
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName1, new PassThroughInterceptor());
    interceptor.put(streamName2, new PassThroughInterceptor());
    List<EventData> singlePartitionEventData1 = MockEventData.generateEventData(numEvents);
    List<EventData> singlePartitionEventData2 = MockEventData.generateEventData(numEvents);
    eventData.put(ssp1, singlePartitionEventData1);
    eventData.put(ssp2, singlePartitionEventData2);
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), String.format("%s,%s", streamName1, streamName2));
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName1), MOCK_ENTITY_1);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName1), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName1), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName1), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName2), MOCK_ENTITY_2);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName2), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName2), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName2), EVENTHUB_KEY);
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory eventHubClientWrapperFactory = new MockEventHubClientManagerFactory(eventData);
    EventHubSystemConsumer consumer = new EventHubSystemConsumer(new EventHubConfig(config), systemName, eventHubClientWrapperFactory, interceptor, testMetrics);
    consumer.register(ssp1, EventHubSystemConsumer.START_OF_STREAM);
    consumer.register(ssp2, EventHubSystemConsumer.START_OF_STREAM);
    consumer.start();
    // Mock received data from EventHub
    eventHubClientWrapperFactory.sendToHandlers(consumer.streamPartitionHandlers);
    Set<SystemStreamPartition> ssps = new HashSet<>();
    ssps.add(ssp1);
    ssps.add(ssp2);
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> results = consumer.poll(ssps, 1000);
    verifyEvents(results.get(ssp1), singlePartitionEventData1);
    verifyEvents(results.get(ssp2), singlePartitionEventData2);
    Assert.assertEquals(testMetrics.getCounters(streamName1).size(), 3);
    Assert.assertEquals(testMetrics.getGauges(streamName1).size(), 2);
    Assert.assertEquals(testMetrics.getCounters(streamName2).size(), 3);
    Assert.assertEquals(testMetrics.getGauges(streamName2).size(), 2);
    Map<String, Counter> counters1 = testMetrics.getCounters(streamName1).stream().collect(Collectors.toMap(Counter::getName, Function.identity()));
    Assert.assertEquals(counters1.get(EventHubSystemConsumer.EVENT_READ_RATE).getCount(), numEvents);
    Assert.assertEquals(counters1.get(EventHubSystemConsumer.READ_ERRORS).getCount(), 0);
    Map<String, Counter> counters2 = testMetrics.getCounters(streamName2).stream().collect(Collectors.toMap(Counter::getName, Function.identity()));
    Assert.assertEquals(counters2.get(EventHubSystemConsumer.EVENT_READ_RATE).getCount(), numEvents);
    Assert.assertEquals(counters2.get(EventHubSystemConsumer.READ_ERRORS).getCount(), 0);
}