use of org.apache.storm.hdfs.bolt.sync.CountSyncPolicy in project metron by apache.
the class HdfsWriter method init.
@Override
public void init(Map stormConfig, TopologyContext topologyContext, WriterConfiguration configurations) {
this.stormConfig = stormConfig;
this.stellarProcessor = new StellarProcessor();
this.fileNameFormat.prepare(stormConfig, topologyContext);
if (syncPolicy != null) {
// if the user has specified the sync policy, we don't want to override their wishes.
syncPolicyCreator = new ClonedSyncPolicyCreator(syncPolicy);
} else {
// if the user has not, then we want to have the sync policy depend on the batch size.
syncPolicyCreator = (source, config) -> new CountSyncPolicy(config == null ? 1 : config.getBatchSize(source));
}
}
use of org.apache.storm.hdfs.bolt.sync.CountSyncPolicy in project storm by apache.
the class TestSequenceFileBolt method makeSeqBolt.
private SequenceFileBolt makeSeqBolt(String nameNodeAddr, int countSync, float rotationSizeMB) {
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
SequenceFormat seqFormat = new DefaultSequenceFormat("key", "value");
return new SequenceFileBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRotationPolicy(fieldsRotationPolicy).withSequenceFormat(seqFormat).withSyncPolicy(fieldsSyncPolicy);
}
use of org.apache.storm.hdfs.bolt.sync.CountSyncPolicy in project storm by apache.
the class TestHdfsBolt method makeHdfsBolt.
private HdfsBolt makeHdfsBolt(String nameNodeAddr, int countSync, float rotationSizeMB) {
RecordFormat fieldsFormat = new DelimitedRecordFormat().withFieldDelimiter("|");
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
return new HdfsBolt().withFsUrl(nameNodeAddr).withFileNameFormat(fieldsFileNameFormat).withRecordFormat(fieldsFormat).withRotationPolicy(fieldsRotationPolicy).withSyncPolicy(fieldsSyncPolicy);
}
use of org.apache.storm.hdfs.bolt.sync.CountSyncPolicy in project metron by apache.
the class SourceHandlerTest method testRotateOutputFile.
@Test
public void testRotateOutputFile() throws IOException {
SourceHandler handler = new SourceHandler(rotActions, // Don't actually care about the rotation
new FileSizeRotationPolicy(10000, Units.MB), new CountSyncPolicy(1), testFormat, callback);
handler.rotateOutputFile();
// Function should ensure rotation actions and callback are called.
verify(rotAction1).execute(any(), any());
verify(rotAction2).execute(any(), any());
verify(callback).removeKey();
}
use of org.apache.storm.hdfs.bolt.sync.CountSyncPolicy in project metron by apache.
the class ClonedSyncPolicyCreatorTest method testClonedPolicy.
@Test
public void testClonedPolicy() {
CountSyncPolicy basePolicy = new CountSyncPolicy(5);
ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy);
// ensure cloned policy continues to work and adheres to the contract: mark on 5th call.
SyncPolicy clonedPolicy = creator.create("blah", null);
for (int i = 0; i < 4; ++i) {
assertFalse(clonedPolicy.mark(null, i));
}
assertTrue(clonedPolicy.mark(null, 5));
// reclone policy and ensure it adheres to the original contract.
clonedPolicy = creator.create("blah", null);
assertFalse(clonedPolicy.mark(null, 0));
}
Aggregations