Examples with MessageId org.apache.metron.common.writer.MessageId used on opensource projects

Example 11 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class SimpleHbaseEnrichmentWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, List<BulkMessage<JSONObject>> messages) throws Exception {
    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
    Table table = getTable(sensorConfig);
    KeyTransformer transformer = getTransformer(sensorConfig);
    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
    String enrichmentType = enrichmentTypeObj == null ? null : enrichmentTypeObj.toString();
    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
    List<Put> puts = new ArrayList<>();
    for (BulkMessage<JSONObject> bulkWriterMessage : messages) {
        EnrichmentKey key = getKey(bulkWriterMessage.getMessage(), transformer, enrichmentType);
        EnrichmentValue value = getValue(bulkWriterMessage.getMessage(), transformer.keySet, valueColumns);
        if (key == null || value == null) {
            continue;
        }
        Put put = converter.toPut(this.cf, key, value);
        if (put != null) {
            LOG.debug("Put: {Column Family: '{}', Key: '{}', Value: '{}'}", this.cf, key, value);
            puts.add(put);
        }
    }
    Set<MessageId> ids = messages.stream().map(BulkMessage::getId).collect(Collectors.toSet());
    BulkWriterResponse response = new BulkWriterResponse();
    try {
        table.put(puts);
    } catch (Exception e) {
        response.addAllErrors(e, ids);
        return response;
    }
    // Can return no errors, because put will throw Exception on error.
    response.addAllSuccesses(ids);
    return response;
}

Example 12 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class AckTuplesPolicyTest method shouldOnlyReportErrorsOncePerBatch.

@Test
public void shouldOnlyReportErrorsOncePerBatch() {
    AckTuplesPolicy ackTuplesPolicy = new AckTuplesPolicy(collector, messageGetStrategy);
    JSONObject rawMessage1 = new JSONObject();
    JSONObject rawMessage2 = new JSONObject();
    rawMessage1.put("value", "rawMessage1");
    rawMessage2.put("value", "rawMessage2");
    String messageId1 = "messageId1";
    String messageId2 = "messageId2";
    String messageId3 = "messageId3";
    JSONObject message1 = new JSONObject();
    JSONObject message2 = new JSONObject();
    JSONObject message3 = new JSONObject();
    message1.put("value", "message1");
    message2.put("value", "message2");
    message3.put("value", "message3");
    Throwable e1 = new Exception("test exception 1");
    Throwable e2 = new Exception("test exception 2");
    MetronError expectedError1 = new MetronError().withSensorType(Collections.singleton(sensorType)).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e1).withRawMessages(Collections.singletonList(rawMessage1));
    MetronError expectedError2 = new MetronError().withSensorType(Collections.singleton(sensorType)).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e2).withRawMessages(Collections.singletonList(rawMessage1));
    MetronError expectedError3 = new MetronError().withSensorType(Collections.singleton(sensorType)).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e1).withRawMessages(Collections.singletonList(rawMessage2));
    when(messageGetStrategy.get(tuple1)).thenReturn(rawMessage1);
    when(messageGetStrategy.get(tuple2)).thenReturn(rawMessage2);
    ackTuplesPolicy.addTupleMessageIds(tuple1, Arrays.asList(messageId1, messageId2));
    ackTuplesPolicy.addTupleMessageIds(tuple2, Collections.singletonList(messageId3));
    BulkWriterResponse response = new BulkWriterResponse();
    response.addError(e1, new MessageId(messageId1));
    ackTuplesPolicy.onFlush(sensorType, response);
    assertEquals(2, ackTuplesPolicy.getTupleMessageMap().size());
    assertEquals(1, ackTuplesPolicy.getTupleErrorMap().size());
    verify(collector, times(0)).ack(any());
    verify(collector, times(0)).reportError(any());
    verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), new Values(argThat(new MetronErrorJSONMatcher(expectedError1.getJSONObject()))));
    response = new BulkWriterResponse();
    response.addError(e2, new MessageId(messageId2));
    response.addError(e1, new MessageId(messageId3));
    ackTuplesPolicy.onFlush(sensorType, response);
    assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
    assertEquals(0, ackTuplesPolicy.getTupleErrorMap().size());
    verify(collector, times(1)).ack(tuple1);
    verify(collector, times(1)).ack(tuple2);
    verify(collector, times(1)).reportError(e1);
    verify(collector, times(1)).reportError(e2);
    verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), new Values(argThat(new MetronErrorJSONMatcher(expectedError2.getJSONObject()))));
    verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), new Values(argThat(new MetronErrorJSONMatcher(expectedError3.getJSONObject()))));
    verifyNoMoreInteractions(collector);
}

Example 13 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class AckTuplesPolicyTest method shouldOnlyAckTupleAfterHandlingAllMessages.

@Test
public void shouldOnlyAckTupleAfterHandlingAllMessages() {
    ackTuplesPolicy.addTupleMessageIds(tuple1, Arrays.asList("message1", "message2", "message3"));
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(new MessageId("message1"));
    response.addSuccess(new MessageId("message2"));
    ackTuplesPolicy.onFlush(sensorType, response);
    verify(collector, times(0)).ack(any());
    response = new BulkWriterResponse();
    response.addSuccess(new MessageId("message3"));
    ackTuplesPolicy.onFlush(sensorType, response);
    assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
    verify(collector, times(1)).ack(tuple1);
    verifyNoMoreInteractions(collector);
}

Example 14 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class AckTuplesPolicyTest method shouldProperlyAckTuples.

@Test
public void shouldProperlyAckTuples() {
    ackTuplesPolicy.addTupleMessageIds(tuple1, Collections.singletonList("message1"));
    ackTuplesPolicy.addTupleMessageIds(tuple2, Collections.singletonList("message2"));
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(new MessageId("message1"));
    response.addSuccess(new MessageId("message2"));
    ackTuplesPolicy.onFlush(sensorType, response);
    assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
    verify(collector, times(1)).ack(tuple1);
    verify(collector, times(1)).ack(tuple2);
    verifyNoMoreInteractions(collector);
}

Example 15 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteSuccessfullyWhenMessageTimestampIsString.

@Test
public void shouldWriteSuccessfullyWhenMessageTimestampIsString() {
    List<BulkMessage<JSONObject>> messages = createMessages(1);
    JSONObject message = messages.get(0).getMessage();
    // the timestamp is a String, rather than a Long
    message.put(Constants.Fields.TIMESTAMP.getName(), new Long(System.currentTimeMillis()).toString());
    // create the document
    String timestamp = (String) message.get(Constants.Fields.TIMESTAMP.getName());
    String guid = (String) message.get(Constants.GUID);
    String sensorType = (String) message.get(Constants.SENSOR_TYPE);
    MessageIdBasedDocument document = new MessageIdBasedDocument(message, guid, sensorType, Long.parseLong(timestamp), new MessageId("message1"));
    // create a document writer which will successfully write that document
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(document);
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
}