Examples with TopologyContext org.apache.storm.task.TopologyContext used on opensource projects

Example 51 with TopologyContext

use of org.apache.storm.task.TopologyContext in project storm by apache.

the class WindowedBoltExecutorTest method testPrepareLateTupleStreamWithoutTs.

@Test
public void testPrepareLateTupleStreamWithoutTs() throws Exception {
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000);
    conf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20);
    conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10);
    conf.put(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late");
    conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5);
    conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 10);
    testWindowedBolt = new TestWindowedBolt();
    executor = new WindowedBoltExecutor(testWindowedBolt);
    TopologyContext context = getTopologyContext();
    // emulate the call of withLateTupleStream method
    Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late")));
    try {
        executor.prepare(conf, context, getOutputCollector());
        fail();
    } catch (IllegalArgumentException e) {
        assertThat(e.getMessage(), is("Late tuple stream can be defined only when specifying a timestamp field"));
    }
}

Example 52 with TopologyContext

use of org.apache.storm.task.TopologyContext in project storm by apache.

the class SpoutWithMockedConsumerSetupHelper method setupSpout.

/**
 * Creates, opens and activates a KafkaSpout using a mocked consumer. The TopicFilter and ManualPartitioner should be mock objects,
 * since this method shortcircuits the TopicPartition assignment process and just calls onPartitionsAssigned on the rebalance listener.
 *
 * @param <K> The Kafka key type
 * @param <V> The Kafka value type
 * @param spoutConfig The spout config to use
 * @param topoConf The topo conf to pass to the spout
 * @param contextMock The topo context to pass to the spout
 * @param collectorMock The mocked collector to pass to the spout
 * @param consumerMock The mocked consumer
 * @param assignedPartitions The partitions to assign to this spout. The consumer will act like these partitions are assigned to it.
 * @return The spout
 */
public static <K, V> KafkaSpout<K, V> setupSpout(KafkaSpoutConfig<K, V> spoutConfig, Map<String, Object> topoConf, TopologyContext contextMock, SpoutOutputCollector collectorMock, KafkaConsumer<K, V> consumerMock, TopicPartition... assignedPartitions) {
    TopicFilter topicFilter = spoutConfig.getTopicFilter();
    ManualPartitioner topicPartitioner = spoutConfig.getTopicPartitioner();
    if (!mockingDetails(topicFilter).isMock() || !mockingDetails(topicPartitioner).isMock()) {
        throw new IllegalStateException("Use a mocked TopicFilter and a mocked ManualPartitioner when using this method, it helps avoid complex stubbing");
    }
    Set<TopicPartition> assignedPartitionsSet = new HashSet<>(Arrays.asList(assignedPartitions));
    TopicAssigner assigner = mock(TopicAssigner.class);
    doAnswer(invocation -> {
        ConsumerRebalanceListener listener = invocation.getArgument(2);
        listener.onPartitionsAssigned(assignedPartitionsSet);
        return null;
    }).when(assigner).assignPartitions(any(), any(), any());
    when(consumerMock.assignment()).thenReturn(assignedPartitionsSet);
    ConsumerFactory<K, V> consumerFactory = (kafkaSpoutConfig) -> consumerMock;
    KafkaSpout<K, V> spout = new KafkaSpout<>(spoutConfig, consumerFactory, assigner);
    spout.open(topoConf, contextMock, collectorMock);
    spout.activate();
    return spout;
}

Example 53 with TopologyContext

use of org.apache.storm.task.TopologyContext in project storm by apache.

the class StreamBuilderTest method testPartitionByKeySinglePartition.

@Test
public void testPartitionByKeySinglePartition() {
    TopologyContext mockContext = Mockito.mock(TopologyContext.class);
    OutputCollector mockCollector = Mockito.mock(OutputCollector.class);
    Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.mapToPair(x -> Pair.of(x, x)).reduceByKey((x, y) -> x + y).print();
    StormTopology topology = streamBuilder.build();
    assertEquals(1, topology.get_bolts_size());
}

Example 54 with TopologyContext

use of org.apache.storm.task.TopologyContext in project storm by apache.

the class StreamBuilderTest method testMultiPartitionByKey.

@Test
public void testMultiPartitionByKey() {
    TopologyContext mockContext = Mockito.mock(TopologyContext.class);
    OutputCollector mockCollector = Mockito.mock(OutputCollector.class);
    Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.mapToPair(x -> Pair.of(x, x)).window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).reduceByKey((x, y) -> x + y).reduceByKey((x, y) -> 0).print();
    StormTopology topology = streamBuilder.build();
    assertEquals(2, topology.get_bolts_size());
}

Example 55 with TopologyContext

use of org.apache.storm.task.TopologyContext in project storm by apache.

the class ExecutorShutdown method shutdown.

@Override
public void shutdown() {
    try {
        LOG.info("Shutting down executor " + executor.getComponentId() + ":" + executor.getExecutorId());
        executor.getReceiveQueue().close();
        for (Utils.SmartThread t : threads) {
            t.interrupt();
        }
        for (Utils.SmartThread t : threads) {
            LOG.debug("Executor " + executor.getComponentId() + ":" + executor.getExecutorId() + " joining thread " + t.getName());
            // Don't wait forever.
            // This is to avoid the deadlock between the executor thread (t) and the shutdown hook (which invokes Worker::shutdown)
            // when it is the executor thread (t) who invokes the shutdown hook. See STORM-3658.
            long waitMs = 100;
            t.join(waitMs);
            if (t.isAlive()) {
                LOG.warn("Thread {} is still alive ({} ms after interruption). Stop waiting for it.", t.getName(), waitMs);
            }
        }
        executor.getStats().cleanupStats();
        for (Task task : taskDatas) {
            if (task == null) {
                continue;
            }
            TopologyContext userContext = task.getUserContext();
            for (ITaskHook hook : userContext.getHooks()) {
                hook.cleanup();
            }
        }
        executor.getStormClusterState().disconnect();
        if (executor.getOpenOrPrepareWasCalled().get()) {
            for (Task task : taskDatas) {
                if (task == null) {
                    continue;
                }
                Object object = task.getTaskObject();
                if (object instanceof ISpout) {
                    ((ISpout) object).close();
                } else if (object instanceof IBolt) {
                    ((IBolt) object).cleanup();
                } else {
                    LOG.error("unknown component object");
                }
            }
        }
        LOG.info("Shut down executor " + executor.getComponentId() + ":" + executor.getExecutorId());
    } catch (Exception e) {
        throw Utils.wrapInRuntime(e);
    }
}