Examples with ValueMapper org.apache.storm.streams.operations.mappers.ValueMapper used on opensource projects

Example 1 with ValueMapper

use of org.apache.storm.streams.operations.mappers.ValueMapper in project storm by apache.

the class StateQueryExample method main.

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2).mapToPair(w -> Pair.of(w, 1)).updateStateByKey(0L, (count, val) -> count + 1);
    /*
         * A stream of words emitted by the QuerySpout is used as
         * the keys to query the state.
         */
    builder.newStream(new QuerySpout(), new ValueMapper<String>(0)).stateQuery(ss).print();
    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    if (args.length > 0) {
        config.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.build());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalCluster.LocalTopology topo = cluster.submitTopology("test", config, builder.build())) {
            Utils.sleep(60_000);
        }
    }
}

Example 2 with ValueMapper

use of org.apache.storm.streams.operations.mappers.ValueMapper in project storm by apache.

the class WordCountToBolt method main.

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    // Redis config parameters for the RedisStoreBolt
    JedisPoolConfig poolConfig = new JedisPoolConfig.Builder().setHost("127.0.0.1").setPort(6379).build();
    // Storm tuple to redis key-value mapper
    RedisStoreMapper storeMapper = new WordCountStoreMapper();
    // The redis bolt (sink)
    IRichBolt redisStoreBolt = new RedisStoreBolt(poolConfig, storeMapper);
    // A stream of words
    builder.newStream(new TestWordSpout(), new ValueMapper<String>(0)).mapToPair(w -> Pair.of(w, 1)).countByKey().to(redisStoreBolt);
    Config config = new Config();
    if (args.length > 0) {
        config.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.build());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalCluster.LocalTopology topo = cluster.submitTopology("test", config, builder.build())) {
            Utils.sleep(60_000);
        }
    }
}

Example 3 with ValueMapper

use of org.apache.storm.streams.operations.mappers.ValueMapper in project storm by apache.

the class StreamBuilderTest method testRepartition.

@Test
public void testRepartition() throws Exception {
    Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>());
    StormTopology topology = streamBuilder.build();
    assertEquals(1, topology.get_spouts_size());
    SpoutSpec spout = topology.get_spouts().get("spout1");
    assertEquals(4, topology.get_bolts_size());
    Bolt bolt1 = topology.get_bolts().get("bolt1");
    Bolt bolt2 = topology.get_bolts().get("bolt2");
    Bolt bolt3 = topology.get_bolts().get("bolt3");
    Bolt bolt4 = topology.get_bolts().get("bolt4");
    assertEquals(1, spout.get_common().get_parallelism_hint());
    assertEquals(1, bolt1.get_common().get_parallelism_hint());
    assertEquals(3, bolt2.get_common().get_parallelism_hint());
    assertEquals(2, bolt3.get_common().get_parallelism_hint());
    assertEquals(2, bolt4.get_common().get_parallelism_hint());
}

Example 4 with ValueMapper

use of org.apache.storm.streams.operations.mappers.ValueMapper in project storm by apache.

the class StreamBuilderTest method testMultiPartitionByKeyWithRepartition.

@Test
public void testMultiPartitionByKeyWithRepartition() {
    TopologyContext mockContext = Mockito.mock(TopologyContext.class);
    OutputCollector mockCollector = Mockito.mock(OutputCollector.class);
    Map<GlobalStreamId, Grouping> expected = new HashMap<>();
    expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key")));
    expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct()));
    Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.mapToPair(x -> Pair.of(x, x)).window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).reduceByKey((x, y) -> x + y).repartition(10).reduceByKey((x, y) -> 0).print();
    StormTopology topology = streamBuilder.build();
    assertEquals(3, topology.get_bolts_size());
    assertEquals(expected, topology.get_bolts().get("bolt3").get_common().get_inputs());
}

Example 5 with ValueMapper

use of org.apache.storm.streams.operations.mappers.ValueMapper in project storm by apache.

the class StreamBuilderTest method testBranchAndJoin.

@Test
public void testBranchAndJoin() throws Exception {
    TopologyContext mockContext = Mockito.mock(TopologyContext.class);
    OutputCollector mockCollector = Mockito.mock(OutputCollector.class);
    Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2);
    Stream<Integer>[] streams = stream.branch(x -> x % 2 == 0, x -> x % 2 == 1);
    PairStream<Integer, Pair<Integer, Integer>> joined = streams[0].mapToPair(x -> Pair.of(x, 1)).join(streams[1].mapToPair(x -> Pair.of(x, 1)));
    assertTrue(joined.getNode() instanceof ProcessorNode);
    StormTopology topology = streamBuilder.build();
    assertEquals(2, topology.get_bolts_size());
}