Examples with WriterConfiguration org.apache.metron.common.configuration.writer.WriterConfiguration used on opensource projects

Example 21 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathMultipleFunctions.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathMultipleFunctions() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key.2", "test.value.2");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s', test.key)", message);
    Assert.assertEquals("test.value", result);
    result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s/%s', test.key, test.key.2)", message);
    Assert.assertEquals("test.value/test.value.2", result);
    result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s', test.key)", message);
    writer.close();
    Assert.assertEquals("test.value", result);
}

Example 22 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathNull.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathNull() {
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, null, message);
    writer.close();
    Assert.assertEquals(SENSOR_NAME, result);
}

Example 23 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetSourceHandlerOpenFilesMax.

@Test
public void testGetSourceHandlerOpenFilesMax() throws IOException {
    int maxFiles = 2;
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat).withMaxOpenFiles(maxFiles);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    for (int i = 0; i < maxFiles; i++) {
        writer.getSourceHandler(SENSOR_NAME, Integer.toString(i), null);
    }
}

Example 24 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testWriteMultipleFiles.

@Test
@SuppressWarnings("unchecked")
public void testWriteMultipleFiles() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    // These two messages will be routed to the same folder, because test.key is the same
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key2", "test.value2");
    JSONObject message2 = new JSONObject();
    message2.put("test.key", "test.value2");
    message2.put("test.key3", "test.value3");
    ArrayList<JSONObject> messages = new ArrayList<>();
    messages.add(message);
    messages.add(message2);
    ArrayList<Tuple> tuples = new ArrayList<>();
    writer.write(SENSOR_NAME, config, tuples, messages);
    writer.close();
    ArrayList<String> expected1 = new ArrayList<>();
    expected1.add(message.toJSONString());
    Collections.sort(expected1);
    File outputFolder1 = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    Assert.assertTrue(outputFolder1.exists() && outputFolder1.isDirectory());
    Assert.assertEquals(1, outputFolder1.listFiles().length);
    for (File file : outputFolder1.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        Assert.assertEquals(expected1, lines);
    }
    ArrayList<String> expected2 = new ArrayList<>();
    expected2.add(message2.toJSONString());
    Collections.sort(expected2);
    File outputFolder2 = new File(folder.getAbsolutePath() + "/test-test.value2/test.value2/");
    Assert.assertTrue(outputFolder2.exists() && outputFolder2.isDirectory());
    Assert.assertEquals(1, outputFolder2.listFiles().length);
    for (File file : outputFolder2.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        Assert.assertEquals(expected2, lines);
    }
}

Example 25 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathDirectVariable.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathDirectVariable() {
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, new IndexingConfigurations());
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "test.key", message);
    writer.close();
    Assert.assertEquals("test.value", result);
}