use of org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy in project storm by apache.
the class TestSequenceFileBolt method makeSeqBolt.
private SequenceFileBolt makeSeqBolt(String nameNodeAddr, int countSync, float rotationSizeMB) {
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
SequenceFormat seqFormat = new DefaultSequenceFormat("key", "value");
return new SequenceFileBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRotationPolicy(fieldsRotationPolicy).withSequenceFormat(seqFormat).withSyncPolicy(fieldsSyncPolicy);
}
use of org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy in project storm by apache.
the class TestHdfsBolt method makeHdfsBolt.
private HdfsBolt makeHdfsBolt(String nameNodeAddr, int countSync, float rotationSizeMB) {
RecordFormat fieldsFormat = new DelimitedRecordFormat().withFieldDelimiter("|");
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
return new HdfsBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRecordFormat(fieldsFormat).withRotationPolicy(fieldsRotationPolicy).withSyncPolicy(fieldsSyncPolicy);
}
use of org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy in project storm by apache.
the class TestHdfsBolt method testCleanupDoesNotThrowExceptionWhenRotationPolicyIsNotTimed.
@Test
public void testCleanupDoesNotThrowExceptionWhenRotationPolicyIsNotTimed() {
// STORM-3372: Rotation policy other than TimedRotationPolicy causes NPE on cleanup
FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(10_000, FileSizeRotationPolicy.Units.MB);
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 10, 10000f).withRotationPolicy(fieldsRotationPolicy);
bolt.prepare(new Config(), topologyContext, collector);
bolt.cleanup();
}
use of org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy in project metron by apache.
the class SourceHandlerTest method testRotateOutputFile.
@Test
public void testRotateOutputFile() throws IOException {
SourceHandler handler = new SourceHandler(rotActions, // Don't actually care about the rotation
new FileSizeRotationPolicy(10000, Units.MB), new CountSyncPolicy(1), testFormat, callback);
handler.rotateOutputFile();
// Function should ensure rotation actions and callback are called.
verify(rotAction1).execute(any(), any());
verify(rotAction2).execute(any(), any());
verify(callback).removeKey();
}
use of org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy in project storm by apache.
the class KafkaHdfsTopo method getTopology.
public static StormTopology getTopology(Map config) {
final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
final int hdfsBatch = getInt(config, HDFS_BATCH, DEFAULT_HDFS_BATCH);
// 1 - Setup Kafka Spout --------
String zkConnString = getStr(config, ZOOKEEPER_URI);
String topicName = getStr(config, KAFKA_TOPIC);
BrokerHosts brokerHosts = new ZkHosts(zkConnString);
SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
spoutConfig.scheme = new StringMultiSchemeWithTopic();
spoutConfig.ignoreZkOffsets = true;
KafkaSpout spout = new KafkaSpout(spoutConfig);
// 2 - Setup HFS Bolt --------
String Hdfs_url = getStr(config, HDFS_URI);
RecordFormat format = new LineWriter("str");
SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
// Instantiate the HdfsBolt
HdfsBolt bolt = new HdfsBolt().withFsUrl(Hdfs_url).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
// 3 - Setup Topology --------
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
Aggregations