use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class HdfsWriterTest method testGetHdfsPathStringReturned.
@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathStringReturned() {
IndexingConfigurations indexingConfig = new IndexingConfigurations();
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
message.put("test.key", "test.value");
Object result = writer.getHdfsPathExtension(SENSOR_NAME, "TO_UPPER(FORMAT(MAP_GET('key', {'key': 'AbC%s'}), test.key))", message);
writer.close();
Assert.assertEquals("ABCTEST.VALUE", result);
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class HdfsWriterTest method testWriteSingleFileWithNull.
@Test
@SuppressWarnings("unchecked")
public void testWriteSingleFileWithNull() throws Exception {
String function = "FORMAT('test-%s/%s', test.key, test.key)";
WriterConfiguration config = buildWriterConfiguration(function);
FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
// These two messages will be routed to the same folder, because test.key is the same
JSONObject message = new JSONObject();
message.put("test.key2", "test.value2");
ArrayList<JSONObject> messages = new ArrayList<>();
messages.add(message);
ArrayList<Tuple> tuples = new ArrayList<>();
writer.write(SENSOR_NAME, config, tuples, messages);
writer.close();
ArrayList<String> expected = new ArrayList<>();
expected.add(message.toJSONString());
Collections.sort(expected);
File outputFolder = new File(folder.getAbsolutePath() + "/test-null/null/");
Assert.assertTrue(outputFolder.exists() && outputFolder.isDirectory());
Assert.assertEquals(1, outputFolder.listFiles().length);
for (File file : outputFolder.listFiles()) {
List<String> lines = Files.readAllLines(file.toPath());
Collections.sort(lines);
Assert.assertEquals(expected, lines);
}
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class HdfsWriterTest method testGetHdfsPathEmptyString.
@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathEmptyString() {
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
Object result = writer.getHdfsPathExtension(SENSOR_NAME, "", message);
writer.close();
Assert.assertEquals(SENSOR_NAME, result);
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class HdfsWriterTest method testGetSourceHandlerOpenFilesOverMax.
@Test(expected = IllegalStateException.class)
public void testGetSourceHandlerOpenFilesOverMax() throws IOException {
int maxFiles = 2;
IndexingConfigurations indexingConfig = new IndexingConfigurations();
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat).withMaxOpenFiles(maxFiles);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
for (int i = 0; i < maxFiles + 1; i++) {
writer.getSourceHandler(SENSOR_NAME, Integer.toString(i), null);
}
}
use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.
the class HdfsWriterTest method testGetHdfsPathNonString.
@Test(expected = IllegalArgumentException.class)
public void testGetHdfsPathNonString() {
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
writer.getHdfsPathExtension(SENSOR_NAME, "{'key':'value'}", message);
}
Aggregations