use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method testWriteNoOutputFunction.
@Test
@SuppressWarnings("unchecked")
public void testWriteNoOutputFunction() throws Exception {
FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
IndexingConfigurations indexingConfig = new IndexingConfigurations();
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
message.put("test.key", "test.value");
message.put("test.key2", "test.value2");
JSONObject message2 = new JSONObject();
message2.put("test.key", "test.value3");
message2.put("test.key2", "test.value2");
ArrayList<JSONObject> messages = new ArrayList<>();
messages.add(message);
messages.add(message2);
ArrayList<Tuple> tuples = new ArrayList<>();
writer.write(SENSOR_NAME, config, tuples, messages);
writer.close();
ArrayList<String> expected = new ArrayList<>();
expected.add(message.toJSONString());
expected.add(message2.toJSONString());
Collections.sort(expected);
// Default to just putting it in the base folder + the sensor name
File outputFolder = new File(folder.getAbsolutePath() + "/" + SENSOR_NAME);
Assert.assertTrue(outputFolder.exists() && outputFolder.isDirectory());
Assert.assertEquals(1, outputFolder.listFiles().length);
for (File file : outputFolder.listFiles()) {
List<String> lines = Files.readAllLines(file.toPath());
Collections.sort(lines);
Assert.assertEquals(expected, lines);
}
}
use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method testSetsCorrectHdfsFilename.
@Test
public void testSetsCorrectHdfsFilename() {
IndexingConfigurations indexingConfig = new IndexingConfigurations();
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
String filename = writer.fileNameFormat.getName(1, 1);
Assert.assertEquals("prefix-Xcom-7-1-1.json", filename);
}
Aggregations