Examples with OutputCollector org.apache.storm.task.OutputCollector used on opensource projects

Example 26 with OutputCollector

use of org.apache.storm.task.OutputCollector in project incubator-pulsar by apache.

the class PulsarBoltTest method setup.

@Override
protected void setup() throws Exception {
    super.internalSetup();
    super.producerBaseSetup();
    pulsarBoltConf = new PulsarBoltConfiguration();
    pulsarBoltConf.setServiceUrl(serviceUrl);
    pulsarBoltConf.setTopic(topic);
    pulsarBoltConf.setTupleToMessageMapper(tupleToMessageMapper);
    pulsarBoltConf.setMetricsTimeIntervalInSecs(60);
    bolt = new PulsarBolt(pulsarBoltConf, new ClientConfiguration());
    mockCollector = new MockOutputCollector();
    OutputCollector collector = new OutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-bolt-" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    bolt.prepare(Maps.newHashMap(), context, collector);
    consumer = pulsarClient.subscribe(topic, subscriptionName);
}

Example 27 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class WindowedBoltExecutorTest method testExecuteWithLateTupleStream.

@Test
public void testExecuteWithLateTupleStream() throws Exception {
    testWindowedBolt = new TestWindowedBolt();
    testWindowedBolt.withTimestampField("ts");
    executor = new WindowedBoltExecutor(testWindowedBolt);
    TopologyContext context = getTopologyContext();
    Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late")));
    OutputCollector outputCollector = Mockito.mock(OutputCollector.class);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000);
    conf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20);
    conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10);
    conf.put(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late");
    conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5);
    // Trigger manually to avoid timing issues
    conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 1_000_000);
    executor.prepare(conf, context, outputCollector);
    long[] timestamps = { 603, 605, 607, 618, 626, 636, 600 };
    List<Tuple> tuples = new ArrayList<>(timestamps.length);
    for (long ts : timestamps) {
        Tuple tuple = getTuple("s1", new Fields("ts"), new Values(ts), "s1Src");
        tuples.add(tuple);
        executor.execute(tuple);
        // Update the watermark to this timestamp
        executor.waterMarkEventGenerator.run();
    }
    System.out.println(testWindowedBolt.tupleWindows);
    Tuple tuple = tuples.get(tuples.size() - 1);
    Mockito.verify(outputCollector).emit("$late", Arrays.asList(tuple), new Values(tuple));
}

Example 28 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class StreamBuilderTest method testPartitionByKeySinglePartition.

@Test
public void testPartitionByKeySinglePartition() {
    TopologyContext mockContext = Mockito.mock(TopologyContext.class);
    OutputCollector mockCollector = Mockito.mock(OutputCollector.class);
    Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.mapToPair(x -> Pair.of(x, x)).reduceByKey((x, y) -> x + y).print();
    StormTopology topology = streamBuilder.build();
    assertEquals(1, topology.get_bolts_size());
}

Example 29 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class StreamBuilderTest method testMultiPartitionByKey.

@Test
public void testMultiPartitionByKey() {
    TopologyContext mockContext = Mockito.mock(TopologyContext.class);
    OutputCollector mockCollector = Mockito.mock(OutputCollector.class);
    Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.mapToPair(x -> Pair.of(x, x)).window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).reduceByKey((x, y) -> x + y).reduceByKey((x, y) -> 0).print();
    StormTopology topology = streamBuilder.build();
    assertEquals(2, topology.get_bolts_size());
}

Example 30 with OutputCollector

use of org.apache.storm.task.OutputCollector in project storm by apache.

the class TestHiveBolt method testNoAcksIfFlushFails.

@Test
public void testNoAcksIfFlushFails() throws Exception {
    JsonRecordHiveMapper mapper = new JsonRecordHiveMapper().withColumnFields(new Fields(colNames1)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(2).withBatchSize(2);
    HiveBolt failingBolt = new FlushFailureHiveBolt(hiveOptions);
    failingBolt.prepare(config, null, new OutputCollector(collector));
    Tuple tuple1 = generateTestTuple(1, "SJC", "Sunnyvale", "CA");
    Tuple tuple2 = generateTestTuple(2, "SFO", "San Jose", "CA");
    failingBolt.execute(tuple1);
    failingBolt.execute(tuple2);
    verify(collector, never()).ack(tuple1);
    verify(collector, never()).ack(tuple2);
    failingBolt.cleanup();
}