Examples with BulkMessage org.apache.metron.common.writer.BulkMessage used on opensource projects

Example 1 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteManySuccessfullyWithSetDocumentId.

@Test
public void shouldWriteManySuccessfullyWithSetDocumentId() {
    when(writerConfiguration.isSetDocumentId("bro")).thenReturn(true);
    when(writerConfiguration.getFieldNameConverter("bro")).thenReturn("NOOP");
    // create a few message ids and the messages associated with the ids
    List<BulkMessage<JSONObject>> messages = createMessages(3);
    // documents should have field converted
    MessageIdBasedDocument document1 = createDocument(messages.get(0));
    MessageIdBasedDocument document2 = createDocument(messages.get(1));
    MessageIdBasedDocument document3 = createDocument(messages.get(2));
    // documents should have guid as documentID
    document1.setDocumentID(document1.getGuid());
    document2.setDocumentID(document1.getGuid());
    document3.setDocumentID(document1.getGuid());
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(document1);
    results.addSuccess(document2);
    results.addSuccess(document3);
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = spy(ElasticsearchWriter.class);
    doReturn(new SimpleDateFormat()).when(esWriter).getIndexFormat(globals());
    doReturn("bro_index").when(esWriter).getIndexName(eq("bro"), any(), eq(writerConfiguration));
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // documents should have metron guid as documentID
    verify(docWriter, times(1)).addDocument(document1, "bro_index");
    verify(docWriter, times(1)).addDocument(document1, "bro_index");
    verify(docWriter, times(1)).addDocument(document1, "bro_index");
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
    assertTrue(response.getSuccesses().contains(new MessageId("message2")));
    assertTrue(response.getSuccesses().contains(new MessageId("message3")));
}

Example 2 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldHandlePartialFailures.

@Test
public void shouldHandlePartialFailures() {
    // create a few message ids and the messages associated with the ids
    int count = 2;
    List<BulkMessage<JSONObject>> messages = createMessages(count);
    Exception cause = new Exception();
    // create a document writer that will fail one and succeed the other
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addFailure(createDocument(messages.get(0)), cause, "error");
    results.addSuccess(createDocument(messages.get(1)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should contain some successes and some failures
    assertEquals(1, response.getSuccesses().size());
    assertEquals(1, response.getErrors().size());
    assertTrue(response.getErrors().get(cause).contains(new MessageId("message1")));
    assertTrue(response.getSuccesses().contains(new MessageId("message2")));
}

Example 3 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldHandleManyWriteFailures.

@Test
public void shouldHandleManyWriteFailures() {
    // create a few message ids and the messages associated with the ids
    int count = 3;
    List<BulkMessage<JSONObject>> messages = createMessages(count);
    Exception cause = new Exception();
    // create a document writer which will fail all writes
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addFailure(createDocument(messages.get(0)), cause, "error");
    results.addFailure(createDocument(messages.get(1)), cause, "error");
    results.addFailure(createDocument(messages.get(2)), cause, "error");
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // the writer response should only contain failures
    assertEquals(0, response.getSuccesses().size());
    assertEquals(1, response.getErrors().size());
    Collection<MessageId> errors = response.getErrors().get(cause);
    assertTrue(errors.contains(new MessageId("message1")));
    assertTrue(errors.contains(new MessageId("message2")));
    assertTrue(errors.contains(new MessageId("message3")));
}

Example 4 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteManySuccessfully.

@Test
public void shouldWriteManySuccessfully() {
    // create a few message ids and the messages associated with the ids
    List<BulkMessage<JSONObject>> messages = createMessages(3);
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(createDocument(messages.get(0)));
    results.addSuccess(createDocument(messages.get(1)));
    results.addSuccess(createDocument(messages.get(2)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
    assertTrue(response.getSuccesses().contains(new MessageId("message2")));
    assertTrue(response.getSuccesses().contains(new MessageId("message3")));
}

Example 5 with BulkMessage

use of org.apache.metron.common.writer.BulkMessage in project metron by apache.

the class SchemaValidationIntegrationTest method test.

public void test(String sensorType) throws Exception {
    SolrComponent component = null;
    try {
        component = createSolrComponent(sensorType);
        component.start();
        component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s", sensorType));
        Map<String, Object> globalConfig = getGlobalConfig(sensorType, component);
        List<BulkMessage<JSONObject>> messages = new ArrayList<>();
        Map<String, Map<String, Object>> index = new HashMap<>();
        int i = 0;
        for (String message : getData(sensorType)) {
            if (message.trim().length() > 0) {
                Map<String, Object> m = JSONUtils.INSTANCE.load(message.trim(), JSONUtils.MAP_SUPPLIER);
                String guid = getGuid(m);
                index.put(guid, m);
                messages.add(new BulkMessage<>(String.format("message%d", ++i), new JSONObject(m)));
            }
        }
        assertTrue(messages.size() > 0);
        SolrWriter solrWriter = new SolrWriter();
        WriterConfiguration writerConfig = new WriterConfiguration() {

            @Override
            public int getBatchSize(String sensorName) {
                return messages.size();
            }

            @Override
            public int getBatchTimeout(String sensorName) {
                return 0;
            }

            @Override
            public List<Integer> getAllConfiguredTimeouts() {
                return new ArrayList<>();
            }

            @Override
            public String getIndex(String sensorName) {
                return sensorType;
            }

            @Override
            public boolean isEnabled(String sensorName) {
                return true;
            }

            @Override
            public Map<String, Object> getSensorConfig(String sensorName) {
                return new HashMap<String, Object>() {

                    {
                        put("index", sensorType);
                        put("batchSize", messages.size());
                        put("enabled", true);
                    }
                };
            }

            @Override
            public Map<String, Object> getGlobalConfig() {
                return globalConfig;
            }

            @Override
            public boolean isDefault(String sensorName) {
                return false;
            }

            @Override
            public String getFieldNameConverter(String sensorName) {
                return null;
            }
        };
        solrWriter.init(null, writerConfig);
        BulkWriterResponse response = solrWriter.write(sensorType, writerConfig, messages);
        assertTrue(response.getErrors().isEmpty());
        for (Map<String, Object> m : component.getAllIndexedDocs(sensorType)) {
            Map<String, Object> expected = index.get(getGuid(m));
            for (Map.Entry<String, Object> field : expected.entrySet()) {
                if (field.getValue() instanceof Collection && ((Collection) field.getValue()).size() == 0) {
                    continue;
                }
                if (m.get(field.getKey()) instanceof Number) {
                    Number n1 = ConversionUtils.convert(field.getValue(), Double.class);
                    Number n2 = (Number) m.get(field.getKey());
                    boolean isSame = Math.abs(n1.doubleValue() - n2.doubleValue()) < 1e-3;
                    if (!isSame) {
                        String s1 = "" + n1.doubleValue();
                        String s2 = "" + n2.doubleValue();
                        isSame = s1.startsWith(s2) || s2.startsWith(s1);
                    }
                    assertTrue(isSame, "Unable to validate " + field.getKey() + ": " + n1 + " != " + n2);
                } else {
                    assertEquals("" + field.getValue(), "" + m.get(field.getKey()), "Unable to find " + field.getKey());
                }
            }
        }
    } finally {
        if (component != null) {
            component.stop();
        }
    }
}