Examples with TopologyContext org.apache.storm.task.TopologyContext used on opensource projects

Example 11 with TopologyContext

use of org.apache.storm.task.TopologyContext in project metron by apache.

the class BulkMessageWriterBoltTest method testFlushOnBatchTimeout.

@Test
public void testFlushOnBatchTimeout() throws Exception {
    FakeClock clock = new FakeClock();
    BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl").withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()).withMessageGetterField("message").withBatchTimeoutDivisor(3);
    bulkMessageWriterBolt.setCuratorFramework(client);
    bulkMessageWriterBolt.setZKCache(cache);
    bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType, new FileInputStream(sampleSensorIndexingConfigPath));
    bulkMessageWriterBolt.declareOutputFields(declarer);
    verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
    Map stormConf = new HashMap();
    when(bulkMessageWriter.getName()).thenReturn("elasticsearch");
    bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector, clock);
    verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(TopologyContext.class), any(WriterConfiguration.class));
    int batchTimeout = bulkMessageWriterBolt.getDefaultBatchTimeout();
    assertEquals(4, batchTimeout);
    tupleList = new ArrayList<>();
    messageList = new ArrayList<>();
    for (int i = 0; i < 3; i++) {
        when(tuple.getValueByField("message")).thenReturn(fullMessageList.get(i));
        tupleList.add(tuple);
        messageList.add(fullMessageList.get(i));
        bulkMessageWriterBolt.execute(tuple);
        verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), eq(messageList));
    }
    clock.elapseSeconds(5);
    when(tuple.getValueByField("message")).thenReturn(fullMessageList.get(3));
    tupleList.add(tuple);
    messageList.add(fullMessageList.get(3));
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllSuccesses(tupleList);
    when(bulkMessageWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), argThat(new MessageListMatcher(messageList)))).thenReturn(response);
    bulkMessageWriterBolt.execute(tuple);
    verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
    verify(outputCollector, times(4)).ack(tuple);
}

Example 12 with TopologyContext

use of org.apache.storm.task.TopologyContext in project metron by apache.

the class ParserBoltTest method testBatchOfFiveWithError.

@Test
public void testBatchOfFiveWithError() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(t1);
    parserBolt.execute(t2);
    parserBolt.execute(t3);
    parserBolt.execute(t4);
    parserBolt.execute(t5);
    verify(outputCollector, times(1)).ack(t1);
    verify(outputCollector, times(1)).ack(t2);
    verify(outputCollector, times(1)).ack(t3);
    verify(outputCollector, times(1)).ack(t4);
    verify(outputCollector, times(1)).ack(t5);
}

Example 13 with TopologyContext

use of org.apache.storm.task.TopologyContext in project metron by apache.

the class ParserBoltTest method testBatchOfOne.

@Test
public void testBatchOfOne() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(t1);
    when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(Collections.singleton(t1)), any())).thenReturn(response);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(t1);
    verify(outputCollector, times(1)).ack(t1);
}

Example 14 with TopologyContext

use of org.apache.storm.task.TopologyContext in project incubator-pulsar by apache.

the class PulsarSpoutTest method testNoSharedConsumer.

@Test
public void testNoSharedConsumer() throws Exception {
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    pulsarSpoutConf.setSharedConsumerEnabled(false);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, new ClientConfiguration(), consumerConf);
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);
    topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 2);
    otherSpout.close();
    topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}

Example 15 with TopologyContext

use of org.apache.storm.task.TopologyContext in project incubator-pulsar by apache.

the class PulsarSpoutTest method testSharedConsumer.

@Test
public void testSharedConsumer() throws Exception {
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, new ClientConfiguration(), consumerConf);
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);
    topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    otherSpout.close();
    topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}