use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method testGetHdfsPathMultipleFunctions.
@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathMultipleFunctions() {
IndexingConfigurations indexingConfig = new IndexingConfigurations();
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
message.put("test.key", "test.value");
message.put("test.key.2", "test.value.2");
Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s', test.key)", message);
Assert.assertEquals("test.value", result);
result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s/%s', test.key, test.key.2)", message);
Assert.assertEquals("test.value/test.value.2", result);
result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s', test.key)", message);
writer.close();
Assert.assertEquals("test.value", result);
}
use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method testGetHdfsPathNull.
@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathNull() {
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
Object result = writer.getHdfsPathExtension(SENSOR_NAME, null, message);
writer.close();
Assert.assertEquals(SENSOR_NAME, result);
}
use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method testGetSourceHandlerOpenFilesMax.
@Test
public void testGetSourceHandlerOpenFilesMax() throws IOException {
int maxFiles = 2;
IndexingConfigurations indexingConfig = new IndexingConfigurations();
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat).withMaxOpenFiles(maxFiles);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
for (int i = 0; i < maxFiles; i++) {
writer.getSourceHandler(SENSOR_NAME, Integer.toString(i), null);
}
}
use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method testGetHdfsPathDirectVariable.
@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathDirectVariable() {
WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
writer.init(new HashMap<String, String>(), createTopologyContext(), config);
JSONObject message = new JSONObject();
message.put("test.key", "test.value");
Object result = writer.getHdfsPathExtension(SENSOR_NAME, "test.key", message);
writer.close();
Assert.assertEquals("test.value", result);
}
use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.
the class HdfsWriterTest method buildWriterConfiguration.
protected WriterConfiguration buildWriterConfiguration(String function) {
IndexingConfigurations indexingConfig = new IndexingConfigurations();
Map<String, Object> sensorIndexingConfig = new HashMap<>();
Map<String, Object> writerIndexingConfig = new HashMap<>();
writerIndexingConfig.put(IndexingConfigurations.OUTPUT_PATH_FUNCTION_CONF, function);
sensorIndexingConfig.put(WRITER_NAME, writerIndexingConfig);
indexingConfig.updateSensorIndexingConfig(SENSOR_NAME, sensorIndexingConfig);
return new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
}
Aggregations