Search in sources :

Example 1 with DruidBeamStateUpdater

use of org.apache.storm.druid.trident.DruidBeamStateUpdater in project storm by apache.

the class SampleDruidBoltTridentTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length == 0) {
        throw new IllegalArgumentException("There should be at least one argument. Run as `SampleDruidBoltTridentTopology <zk-url>`");
    }
    TridentTopology tridentTopology = new TridentTopology();
    DruidBeamFactory druidBeamFactory = new SampleDruidBeamFactoryImpl(new HashMap<String, Object>());
    ITupleDruidEventMapper<Map<String, Object>> eventMapper = new TupleDruidEventMapper<>(TupleDruidEventMapper.DEFAULT_FIELD_NAME);
    final Stream stream = tridentTopology.newStream("batch-event-gen", new SimpleBatchSpout(10));
    stream.peek(new Consumer() {

        @Override
        public void accept(TridentTuple input) {
            LOG.info("########### Received tuple: [{}]", input);
        }
    }).partitionPersist(new DruidBeamStateFactory<Map<String, Object>>(druidBeamFactory, eventMapper), new Fields("event"), new DruidBeamStateUpdater());
    Config conf = new Config();
    conf.setDebug(true);
    conf.put("druid.tranquility.zk.connect", args[0]);
    if (args.length > 1) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[1], conf, tridentTopology.build());
    } else {
        conf.setMaxTaskParallelism(3);
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("druid-test", conf, tridentTopology.build())) {
            Thread.sleep(30000);
        }
        System.exit(0);
    }
}
Also used : LocalCluster(org.apache.storm.LocalCluster) Config(org.apache.storm.Config) DruidBeamFactory(org.apache.storm.druid.bolt.DruidBeamFactory) DruidBeamStateUpdater(org.apache.storm.druid.trident.DruidBeamStateUpdater) LocalTopology(org.apache.storm.LocalCluster.LocalTopology) Fields(org.apache.storm.tuple.Fields) Consumer(org.apache.storm.trident.operation.Consumer) TridentTopology(org.apache.storm.trident.TridentTopology) TupleDruidEventMapper(org.apache.storm.druid.bolt.TupleDruidEventMapper) ITupleDruidEventMapper(org.apache.storm.druid.bolt.ITupleDruidEventMapper) Stream(org.apache.storm.trident.Stream) HashMap(java.util.HashMap) Map(java.util.Map) TridentTuple(org.apache.storm.trident.tuple.TridentTuple)

Aggregations

HashMap (java.util.HashMap)1 Map (java.util.Map)1 Config (org.apache.storm.Config)1 LocalCluster (org.apache.storm.LocalCluster)1 LocalTopology (org.apache.storm.LocalCluster.LocalTopology)1 DruidBeamFactory (org.apache.storm.druid.bolt.DruidBeamFactory)1 ITupleDruidEventMapper (org.apache.storm.druid.bolt.ITupleDruidEventMapper)1 TupleDruidEventMapper (org.apache.storm.druid.bolt.TupleDruidEventMapper)1 DruidBeamStateUpdater (org.apache.storm.druid.trident.DruidBeamStateUpdater)1 Stream (org.apache.storm.trident.Stream)1 TridentTopology (org.apache.storm.trident.TridentTopology)1 Consumer (org.apache.storm.trident.operation.Consumer)1 TridentTuple (org.apache.storm.trident.tuple.TridentTuple)1 Fields (org.apache.storm.tuple.Fields)1