Examples with WriterConfiguration org.apache.metron.common.configuration.writer.WriterConfiguration used on opensource projects

Example 26 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testHandleAttemptsRotateIfStreamClosed.

@Test
@SuppressWarnings("unchecked")
public void testHandleAttemptsRotateIfStreamClosed() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    ArrayList<JSONObject> messages = new ArrayList<>();
    messages.add(message);
    ArrayList<Tuple> tuples = new ArrayList<>();
    CountSyncPolicy basePolicy = new CountSyncPolicy(5);
    ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy);
    writer.write(SENSOR_NAME, config, tuples, messages);
    writer.getSourceHandler(SENSOR_NAME, "test-test.value/test.value", config).closeOutputFile();
    writer.getSourceHandler(SENSOR_NAME, "test-test.value/test.value", config).handle(message, SENSOR_NAME, config, creator);
    writer.close();
    File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    // The message should show up twice, once in each file
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    // Assert this went into a new file because it actually rotated
    Assert.assertEquals(2, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        // One line per file
        Assert.assertEquals(1, lines.size());
        Assert.assertEquals(expected, lines);
    }
}

Example 27 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testWriteNoOutputFunction.

@Test
@SuppressWarnings("unchecked")
public void testWriteNoOutputFunction() throws Exception {
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key2", "test.value2");
    JSONObject message2 = new JSONObject();
    message2.put("test.key", "test.value3");
    message2.put("test.key2", "test.value2");
    ArrayList<JSONObject> messages = new ArrayList<>();
    messages.add(message);
    messages.add(message2);
    ArrayList<Tuple> tuples = new ArrayList<>();
    writer.write(SENSOR_NAME, config, tuples, messages);
    writer.close();
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    expected.add(message2.toJSONString());
    Collections.sort(expected);
    // Default to just putting it in the base folder + the sensor name
    File outputFolder = new File(folder.getAbsolutePath() + "/" + SENSOR_NAME);
    Assert.assertTrue(outputFolder.exists() && outputFolder.isDirectory());
    Assert.assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        Assert.assertEquals(expected, lines);
    }
}

Example 28 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testSingleFileIfNoStreamClosed.

@Test
@SuppressWarnings("unchecked")
public void testSingleFileIfNoStreamClosed() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    ArrayList<JSONObject> messages = new ArrayList<>();
    messages.add(message);
    ArrayList<Tuple> tuples = new ArrayList<>();
    CountSyncPolicy basePolicy = new CountSyncPolicy(5);
    ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy);
    writer.write(SENSOR_NAME, config, tuples, messages);
    writer.write(SENSOR_NAME, config, tuples, messages);
    writer.close();
    File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    // The message should show up twice, once in each file
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    expected.add(message.toJSONString());
    // Assert both messages are in the same file, because the stream stayed open
    Assert.assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        // One line per file
        Assert.assertEquals(2, lines.size());
        Assert.assertEquals(expected, lines);
    }
}

Example 29 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testSetsCorrectHdfsFilename.

@Test
public void testSetsCorrectHdfsFilename() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    String filename = writer.fileNameFormat.getName(1, 1);
    Assert.assertEquals("prefix-Xcom-7-1-1.json", filename);
}

Example 30 with WriterConfiguration

use of org.apache.metron.common.configuration.writer.WriterConfiguration in project metron by apache.

the class HdfsWriterTest method testWriteSingleFile.

@Test
@SuppressWarnings("unchecked")
public void testWriteSingleFile() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    // These two messages will be routed to the same folder, because test.key is the same
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key2", "test.value2");
    JSONObject message2 = new JSONObject();
    message2.put("test.key", "test.value");
    message2.put("test.key3", "test.value2");
    ArrayList<JSONObject> messages = new ArrayList<>();
    messages.add(message);
    messages.add(message2);
    ArrayList<Tuple> tuples = new ArrayList<>();
    writer.write(SENSOR_NAME, config, tuples, messages);
    writer.close();
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    expected.add(message2.toJSONString());
    Collections.sort(expected);
    File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    Assert.assertTrue(outputFolder.exists() && outputFolder.isDirectory());
    Assert.assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        Assert.assertEquals(expected, lines);
    }
}