Examples with SyncPolicy org.apache.storm.hdfs.bolt.sync.SyncPolicy used on opensource projects

Example 6 with SyncPolicy

use of org.apache.storm.hdfs.bolt.sync.SyncPolicy in project storm by apache.

the class StrGenSpoutHdfsBoltTopo method getTopology.

public static StormTopology getTopology(Map topoConf) {
    final int hdfsBatch = Helper.getInt(topoConf, HDFS_BATCH, DEFAULT_HDFS_BATCH);
    // 1 -  Setup StringGen Spout   --------
    StringGenSpout spout = new StringGenSpout(100).withFieldName("str");
    // 2 -  Setup HFS Bolt   --------
    String Hdfs_url = Helper.getStr(topoConf, HDFS_URI);
    RecordFormat format = new LineWriter("str");
    SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
    final int spoutNum = Helper.getInt(topoConf, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = Helper.getInt(topoConf, BOLT_NUM, DEFAULT_BOLT_NUM);
    // Use default, Storm-generated file names
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(Helper.getStr(topoConf, HDFS_PATH));
    // Instantiate the HdfsBolt
    HdfsBolt bolt = new HdfsBolt().withFsUrl(Hdfs_url).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}

Example 7 with SyncPolicy

use of org.apache.storm.hdfs.bolt.sync.SyncPolicy in project storm by apache.

the class HdfsFileTopology method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);
    SentenceSpout spout = new SentenceSpout();
    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);
    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/foo/").withExtension(".txt");
    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter("|");
    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    config.put("hdfs.config", yamlConf);
    HdfsBolt bolt = new HdfsBolt().withConfigKey("hdfs.config").withFsUrl(args[0]).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy).addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4).shuffleGrouping(SENTENCE_SPOUT_ID);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology())) {
            waitForSeconds(120);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
    }
}

Example 8 with SyncPolicy

use of org.apache.storm.hdfs.bolt.sync.SyncPolicy in project storm by apache.

the class SequenceFileTopology method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);
    SentenceSpout spout = new SentenceSpout();
    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);
    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/source/").withExtension(".seq");
    // create sequence format instance.
    DefaultSequenceFormat format = new DefaultSequenceFormat("timestamp", "sentence");
    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    config.put("hdfs.config", yamlConf);
    SequenceFileBolt bolt = new SequenceFileBolt().withFsUrl(args[0]).withConfigKey("hdfs.config").withFileNameFormat(fileNameFormat).withSequenceFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy).withCompressionType(SequenceFile.CompressionType.RECORD).withCompressionCodec("deflate").addRotationAction(new MoveFileAction().toDestination("/tmp/dest/"));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4).shuffleGrouping(SENTENCE_SPOUT_ID);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology())) {
            waitForSeconds(120);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
    }
}