Search in sources :

Example 1 with FileNameFormat

use of org.apache.storm.hdfs.bolt.format.FileNameFormat in project storm by apache.

the class TestHdfsBolt method makeHdfsBolt.

private HdfsBolt makeHdfsBolt(String nameNodeAddr, int countSync, float rotationSizeMB) {
    RecordFormat fieldsFormat = new DelimitedRecordFormat().withFieldDelimiter("|");
    SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
    FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
    FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
    return new HdfsBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRecordFormat(fieldsFormat).withRotationPolicy(fieldsRotationPolicy).withSyncPolicy(fieldsSyncPolicy);
}
Also used : DelimitedRecordFormat(org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat) RecordFormat(org.apache.storm.hdfs.bolt.format.RecordFormat) DelimitedRecordFormat(org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) FileSizeRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat)

Example 2 with FileNameFormat

use of org.apache.storm.hdfs.bolt.format.FileNameFormat in project storm by apache.

the class TestSequenceFileBolt method makeSeqBolt.

private SequenceFileBolt makeSeqBolt(String nameNodeAddr, int countSync, float rotationSizeMB) {
    SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
    FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
    FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
    SequenceFormat seqFormat = new DefaultSequenceFormat("key", "value");
    return new SequenceFileBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRotationPolicy(fieldsRotationPolicy).withSequenceFormat(seqFormat).withSyncPolicy(fieldsSyncPolicy);
}
Also used : SequenceFormat(org.apache.storm.hdfs.bolt.format.SequenceFormat) DefaultSequenceFormat(org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) FileSizeRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) DefaultSequenceFormat(org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat)

Example 3 with FileNameFormat

use of org.apache.storm.hdfs.bolt.format.FileNameFormat in project storm by apache.

the class HdfsFileTopology method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);
    SentenceSpout spout = new SentenceSpout();
    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);
    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/foo/").withExtension(".txt");
    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter("|");
    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    config.put("hdfs.config", yamlConf);
    HdfsBolt bolt = new HdfsBolt().withConfigKey("hdfs.config").withFsUrl(args[0]).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy).addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4).shuffleGrouping(SENTENCE_SPOUT_ID);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology())) {
            waitForSeconds(120);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
    }
}
Also used : LocalCluster(org.apache.storm.LocalCluster) DelimitedRecordFormat(org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Config(org.apache.storm.Config) RecordFormat(org.apache.storm.hdfs.bolt.format.RecordFormat) DelimitedRecordFormat(org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) TimedRotationPolicy(org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) Yaml(org.yaml.snakeyaml.Yaml) FileInputStream(java.io.FileInputStream) LocalTopology(org.apache.storm.LocalCluster.LocalTopology) MoveFileAction(org.apache.storm.hdfs.common.rotation.MoveFileAction) HashMap(java.util.HashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 4 with FileNameFormat

use of org.apache.storm.hdfs.bolt.format.FileNameFormat in project storm by apache.

the class AvroGenericRecordBoltTest method makeAvroBolt.

private AvroGenericRecordBolt makeAvroBolt(String nameNodeAddr, int countSync, float rotationSizeMB, String schemaAsString) {
    SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
    FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
    return new AvroGenericRecordBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRotationPolicy(rotationPolicy).withSyncPolicy(fieldsSyncPolicy);
}
Also used : CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileSizeRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy)

Example 5 with FileNameFormat

use of org.apache.storm.hdfs.bolt.format.FileNameFormat in project storm by apache.

the class KafkaHdfsTopo method getTopology.

public static StormTopology getTopology(Map config) {
    final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    final int hdfsBatch = getInt(config, HDFS_BATCH, DEFAULT_HDFS_BATCH);
    // 1 -  Setup Kafka Spout   --------
    String zkConnString = getStr(config, ZOOKEEPER_URI);
    String topicName = getStr(config, KAFKA_TOPIC);
    BrokerHosts brokerHosts = new ZkHosts(zkConnString);
    SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
    spoutConfig.scheme = new StringMultiSchemeWithTopic();
    spoutConfig.ignoreZkOffsets = true;
    KafkaSpout spout = new KafkaSpout(spoutConfig);
    // 2 -  Setup HFS Bolt   --------
    String Hdfs_url = getStr(config, HDFS_URI);
    RecordFormat format = new LineWriter("str");
    SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
    // Instantiate the HdfsBolt
    HdfsBolt bolt = new HdfsBolt().withFsUrl(Hdfs_url).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}
Also used : TopologyBuilder(org.apache.storm.topology.TopologyBuilder) RecordFormat(org.apache.storm.hdfs.bolt.format.RecordFormat) SpoutConfig(org.apache.storm.kafka.SpoutConfig) ZkHosts(org.apache.storm.kafka.ZkHosts) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) StringMultiSchemeWithTopic(org.apache.storm.kafka.StringMultiSchemeWithTopic) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) BrokerHosts(org.apache.storm.kafka.BrokerHosts) HdfsBolt(org.apache.storm.hdfs.bolt.HdfsBolt) KafkaSpout(org.apache.storm.kafka.KafkaSpout) FileSizeRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy)

Aggregations

DefaultFileNameFormat (org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat)6 FileNameFormat (org.apache.storm.hdfs.bolt.format.FileNameFormat)6 FileRotationPolicy (org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy)6 CountSyncPolicy (org.apache.storm.hdfs.bolt.sync.CountSyncPolicy)6 SyncPolicy (org.apache.storm.hdfs.bolt.sync.SyncPolicy)6 FileSizeRotationPolicy (org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy)5 RecordFormat (org.apache.storm.hdfs.bolt.format.RecordFormat)4 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)3 HdfsBolt (org.apache.storm.hdfs.bolt.HdfsBolt)2 DelimitedRecordFormat (org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat)2 FileInputStream (java.io.FileInputStream)1 InputStream (java.io.InputStream)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 Config (org.apache.storm.Config)1 LocalCluster (org.apache.storm.LocalCluster)1 LocalTopology (org.apache.storm.LocalCluster.LocalTopology)1 DefaultSequenceFormat (org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat)1 SequenceFormat (org.apache.storm.hdfs.bolt.format.SequenceFormat)1