Examples with HdfsSpout org.apache.storm.hdfs.spout.HdfsSpout used on opensource projects

Example 1 with HdfsSpout

use of org.apache.storm.hdfs.spout.HdfsSpout in project storm by apache.

the class HdfsSpoutNullBoltTopo method getTopology.

public static StormTopology getTopology(Map config) {
    final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    final String fileFormat = Helper.getStr(config, "text");
    final String hdfsUri = Helper.getStr(config, HDFS_URI);
    final String sourceDir = Helper.getStr(config, SOURCE_DIR);
    final String archiveDir = Helper.getStr(config, ARCHIVE_DIR);
    final String badDir = Helper.getStr(config, BAD_DIR);
    // 1 -  Setup Hdfs Spout   --------
    HdfsSpout spout = new HdfsSpout().setReaderType(fileFormat).setHdfsUri(hdfsUri).setSourceDir(sourceDir).setArchiveDir(archiveDir).setBadFilesDir(badDir).withOutputFields(TextFileReader.defaultFields);
    // 2 -   DevNull Bolt   --------
    DevNullBolt bolt = new DevNullBolt();
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}