Examples with BulkMessage org.apache.metron.common.writer.BulkMessage used on opensource projects

Example 16 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccess.

@Test
public void shouldExecuteOnSuccess() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    JSONObject message = new JSONObject();
    message.put(Constants.GUID, "messageId");
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Collections.singletonList("messageId"));
        verify(writerHandler, times(1)).write("yaf", new BulkMessage<>("messageId", message), parserConfigurations);
    }
}

Example 17 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccessWithMultipleMessages.

@Test
public void shouldExecuteOnSuccessWithMultipleMessages() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    List<BulkMessage<JSONObject>> messages = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        String messageId = String.format("messageId%s", i + 1);
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("field", String.format("value%s", i + 1));
        messages.add(new BulkMessage<>(messageId, message));
    }
    mockParserRunner.setMessages(messages.stream().map(BulkMessage::getMessage).collect(Collectors.toList()));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        // Verify the correct message is written and ack is handled
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        InOrder inOrder = inOrder(bulkWriterResponseHandler, writerHandler);
        inOrder.verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Arrays.asList("messageId1", "messageId2", "messageId3", "messageId4", "messageId5"));
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(0), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(1), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(2), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(3), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(4), parserConfigurations);
    }
    verifyNoMoreInteractions(writerHandler, bulkWriterResponseHandler, outputCollector);
}

Example 18 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class HdfsWriterTest method testWriteMultipleFiles.

@Test
@SuppressWarnings("unchecked")
public void testWriteMultipleFiles() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    // These two messages will be routed to the same folder, because test.key is the same
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key2", "test.value2");
    JSONObject message2 = new JSONObject();
    message2.put("test.key", "test.value2");
    message2.put("test.key3", "test.value3");
    List<BulkMessage<JSONObject>> messages = new ArrayList<BulkMessage<JSONObject>>() {

        {
            add(new BulkMessage("message1", message));
            add(new BulkMessage("message2", message2));
        }
    };
    writer.write(SENSOR_NAME, config, messages);
    writer.close();
    ArrayList<String> expected1 = new ArrayList<>();
    expected1.add(message.toJSONString());
    Collections.sort(expected1);
    File outputFolder1 = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    assertTrue(outputFolder1.exists() && outputFolder1.isDirectory());
    assertEquals(1, outputFolder1.listFiles().length);
    for (File file : outputFolder1.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        assertEquals(expected1, lines);
    }
    ArrayList<String> expected2 = new ArrayList<>();
    expected2.add(message2.toJSONString());
    Collections.sort(expected2);
    File outputFolder2 = new File(folder.getAbsolutePath() + "/test-test.value2/test.value2/");
    assertTrue(outputFolder2.exists() && outputFolder2.isDirectory());
    assertEquals(1, outputFolder2.listFiles().length);
    for (File file : outputFolder2.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        assertEquals(expected2, lines);
    }
}

Example 19 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class HdfsWriterTest method testWriteSingleFile.

@Test
@SuppressWarnings("unchecked")
public void testWriteSingleFile() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    // These two messages will be routed to the same folder, because test.key is the same
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key2", "test.value2");
    JSONObject message2 = new JSONObject();
    message2.put("test.key", "test.value");
    message2.put("test.key3", "test.value2");
    List<BulkMessage<JSONObject>> messages = new ArrayList<BulkMessage<JSONObject>>() {

        {
            add(new BulkMessage<>("message1", message));
            add(new BulkMessage<>("message2", message2));
        }
    };
    writer.write(SENSOR_NAME, config, messages);
    writer.close();
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    expected.add(message2.toJSONString());
    Collections.sort(expected);
    File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    assertTrue(outputFolder.exists() && outputFolder.isDirectory());
    assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        assertEquals(expected, lines);
    }
}

Example 20 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class HdfsWriterTest method testSingleFileIfNoStreamClosed.

@Test
@SuppressWarnings("unchecked")
public void testSingleFileIfNoStreamClosed() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    List<BulkMessage<JSONObject>> messages = new ArrayList<BulkMessage<JSONObject>>() {

        {
            add(new BulkMessage("message1", message));
        }
    };
    CountSyncPolicy basePolicy = new CountSyncPolicy(5);
    ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy);
    writer.write(SENSOR_NAME, config, messages);
    writer.write(SENSOR_NAME, config, messages);
    writer.close();
    File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
    // The message should show up twice, once in each file
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    expected.add(message.toJSONString());
    // Assert both messages are in the same file, because the stream stayed open
    assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        // One line per file
        assertEquals(2, lines.size());
        assertEquals(expected, lines);
    }
}