Examples with BulkMessage org.apache.metron.common.writer.BulkMessage used on opensource projects

Example 11 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class HdfsWriterTest method testWriteSingleFileWithNull.

@Test
@SuppressWarnings("unchecked")
public void testWriteSingleFileWithNull() throws Exception {
    String function = "FORMAT('test-%s/%s', test.key, test.key)";
    WriterConfiguration config = buildWriterConfiguration(function);
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    // These two messages will be routed to the same folder, because test.key is the same
    JSONObject message = new JSONObject();
    message.put("test.key2", "test.value2");
    List<BulkMessage<JSONObject>> messages = new ArrayList<BulkMessage<JSONObject>>() {

        {
            add(new BulkMessage("message1", message));
        }
    };
    writer.write(SENSOR_NAME, config, messages);
    writer.close();
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    Collections.sort(expected);
    File outputFolder = new File(folder.getAbsolutePath() + "/test-null/null/");
    assertTrue(outputFolder.exists() && outputFolder.isDirectory());
    assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        assertEquals(expected, lines);
    }
}

Example 12 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteSuccessfullyWhenMessageTimestampIsString.

@Test
public void shouldWriteSuccessfullyWhenMessageTimestampIsString() {
    List<BulkMessage<JSONObject>> messages = createMessages(1);
    JSONObject message = messages.get(0).getMessage();
    // the timestamp is a String, rather than a Long
    message.put(Constants.Fields.TIMESTAMP.getName(), new Long(System.currentTimeMillis()).toString());
    // create the document
    String timestamp = (String) message.get(Constants.Fields.TIMESTAMP.getName());
    String guid = (String) message.get(Constants.GUID);
    String sensorType = (String) message.get(Constants.SENSOR_TYPE);
    MessageIdBasedDocument document = new MessageIdBasedDocument(message, guid, sensorType, Long.parseLong(timestamp), new MessageId("message1"));
    // create a document writer which will successfully write that document
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(document);
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
}

Example 13 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldHandleWriteFailure.

@Test
public void shouldHandleWriteFailure() {
    // create a message id and a message associated with that id
    List<BulkMessage<JSONObject>> messages = createMessages(3);
    Exception cause = new Exception();
    // create a document writer which will fail all writes
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addFailure(createDocument(messages.get(0)), cause, "error");
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // the writer response should only contain failures
    assertEquals(0, response.getSuccesses().size());
    assertEquals(1, response.getErrors().size());
    Collection<MessageId> errors = response.getErrors().get(cause);
    assertTrue(errors.contains(new MessageId("message1")));
}

Example 14 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteSuccessfullyWhenMissingGUID.

@Test
public void shouldWriteSuccessfullyWhenMissingGUID() {
    // create a message id and a message associated with that tuple
    List<BulkMessage<JSONObject>> messages = createMessages(1);
    // remove the GUID from the message
    assertNotNull(messages.get(0).getMessage().remove(Constants.GUID));
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(createDocument(messages.get(0)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
}

Example 15 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteSuccessfully.

@Test
public void shouldWriteSuccessfully() {
    // create a message id and a message associated with that id
    List<BulkMessage<JSONObject>> messages = createMessages(1);
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(createDocument(messages.get(0)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
}