Examples with BulkWriterResponse org.apache.metron.common.writer.BulkWriterResponse used on opensource projects

Example 16 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class ElasticsearchWriterTest method testTwoSameFailure.

@Test
public void testTwoSameFailure() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    Tuple tuple2 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    Exception e = new IllegalStateException();
    BulkItemResponse itemResponse = buildBulkItemFailure(e);
    BulkItemResponse itemResponse2 = buildBulkItemFailure(e);
    when(response.iterator()).thenReturn(ImmutableList.of(itemResponse, itemResponse2).iterator());
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addError(e, tuple1);
    expected.addError(e, tuple2);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1, tuple2), response);
    assertEquals("Response should have two errors and no successes", expected, actual);
    // Ensure the errors actually get collapsed together
    Map<Throwable, Collection<Tuple>> actualErrors = actual.getErrors();
    HashMap<Throwable, Collection<Tuple>> expectedErrors = new HashMap<>();
    expectedErrors.put(e, ImmutableList.of(tuple1, tuple2));
    assertEquals("Errors should have collapsed together", expectedErrors, actualErrors);
}

Example 17 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class ElasticsearchWriterTest method testTwoDifferentFailure.

@Test
public void testTwoDifferentFailure() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    Tuple tuple2 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    Exception e = new IllegalStateException("Cause");
    Exception e2 = new IllegalStateException("Different Cause");
    BulkItemResponse itemResponse = buildBulkItemFailure(e);
    BulkItemResponse itemResponse2 = buildBulkItemFailure(e2);
    when(response.iterator()).thenReturn(ImmutableList.of(itemResponse, itemResponse2).iterator());
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addError(e, tuple1);
    expected.addError(e2, tuple2);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1, tuple2), response);
    assertEquals("Response should have two errors and no successes", expected, actual);
    // Ensure the errors did not get collapsed together
    Map<Throwable, Collection<Tuple>> actualErrors = actual.getErrors();
    HashMap<Throwable, Collection<Tuple>> expectedErrors = new HashMap<>();
    expectedErrors.put(e, ImmutableList.of(tuple1));
    expectedErrors.put(e2, ImmutableList.of(tuple2));
    assertEquals("Errors should not have collapsed together", expectedErrors, actualErrors);
}

Example 18 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class BulkWriterComponent method flush.

private void flush(String sensorType, BulkMessageWriter<MESSAGE_T> bulkMessageWriter, WriterConfiguration configurations, MessageGetStrategy messageGetStrategy, Collection<Tuple> tupleList, List<MESSAGE_T> messageList) throws Exception {
    // no need to mock, so use real time
    long startTime = System.currentTimeMillis();
    try {
        BulkWriterResponse response = bulkMessageWriter.write(sensorType, configurations, tupleList, messageList);
        // Commit or error piecemeal.
        if (handleCommit) {
            commit(response);
        }
        if (handleError) {
            error(sensorType, response, messageGetStrategy);
        } else if (response.hasErrors()) {
            throw new IllegalStateException("Unhandled bulk errors in response: " + response.getErrors());
        }
    } catch (Throwable e) {
        if (handleError) {
            error(sensorType, e, tupleList, messageGetStrategy);
        } else {
            throw e;
        }
    } finally {
        sensorTupleMap.remove(sensorType);
        sensorMessageMap.remove(sensorType);
    }
    long endTime = System.currentTimeMillis();
    long elapsed = endTime - startTime;
    LOG.debug("Bulk batch for sensor {} completed in ~{} ns", sensorType, elapsed);
}

Example 19 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class WriterToBulkWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<MESSAGE_T> messages) throws Exception {
    BulkWriterResponse response = new BulkWriterResponse();
    if (messages.size() > 1) {
        response.addAllErrors(new IllegalStateException("WriterToBulkWriter expects a batch of exactly 1"), tuples);
        return response;
    }
    try {
        messageWriter.write(sensorType, configurations, Iterables.getFirst(tuples, null), Iterables.getFirst(messages, null));
    } catch (Exception e) {
        response.addAllErrors(e, tuples);
        return response;
    }
    response.addAllSuccesses(tuples);
    return response;
}

Example 20 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class BulkWriterComponentTest method writeShouldProperlyAckTuplesInBatch.

@Test
public void writeShouldProperlyAckTuplesInBatch() throws Exception {
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllSuccesses(tupleList);
    when(bulkMessageWriter.write(sensorType, configurations, Arrays.asList(tuple1, tuple2), Arrays.asList(message1, message2))).thenReturn(response);
    BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(collector);
    bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
    verify(bulkMessageWriter, times(0)).write(sensorType, configurations, Collections.singletonList(tuple1), Collections.singletonList(message1));
    verify(collector, times(0)).ack(tuple1);
    verify(collector, times(0)).ack(tuple2);
    bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy);
    verify(collector, times(1)).ack(tuple1);
    verify(collector, times(1)).ack(tuple2);
    // A disabled writer should still ack
    Tuple disabledTuple = mock(Tuple.class);
    String disabledSensorType = "disabled";
    when(configurations.isEnabled(disabledSensorType)).thenReturn(false);
    bulkWriterComponent.write(disabledSensorType, disabledTuple, message2, bulkMessageWriter, configurations, messageGetStrategy);
    verify(collector, times(1)).ack(disabledTuple);
    verifyStatic(times(0));
    ErrorUtils.handleError(eq(collector), any(MetronError.class));
}