Examples with BulkWriterResponse org.apache.metron.common.writer.BulkWriterResponse used on opensource projects

Example 1 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class ElasticsearchWriterTest method testSingleFailure.

@Test
public void testSingleFailure() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    Exception e = new IllegalStateException();
    BulkItemResponse itemResponse = buildBulkItemFailure(e);
    when(response.iterator()).thenReturn(ImmutableList.of(itemResponse).iterator());
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addError(e, tuple1);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1), response);
    assertEquals("Response should have one error and zero successes", expected, actual);
}

Example 2 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class SolrWriter method write.

@Override
public BulkWriterResponse write(String sourceType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    for (JSONObject message : messages) {
        SolrInputDocument document = new SolrInputDocument();
        document.addField("id", getIdValue(message));
        document.addField("sensorType", sourceType);
        for (Object key : message.keySet()) {
            Object value = message.get(key);
            document.addField(getFieldName(key, value), value);
        }
        UpdateResponse response = solr.add(document);
    }
    if (shouldCommit) {
        solr.commit(getCollection(configurations));
    }
    // Solr commits the entire batch or throws an exception for it.  There's no way to get partial failures.
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllSuccesses(tuples);
    return response;
}

Example 3 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class NoopWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    if (sleepFunction != null) {
        sleepFunction.apply(null);
    }
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllSuccesses(tuples);
    return response;
}

Example 4 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class HdfsWriter method write.

@Override
public BulkWriterResponse write(String sourceType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    BulkWriterResponse response = new BulkWriterResponse();
    // Currently treating all the messages in a group for pass/failure.
    try {
        // Messages can all result in different HDFS paths, because of Stellar Expressions, so we'll need to iterate through
        for (JSONObject message : messages) {
            String path = getHdfsPathExtension(sourceType, (String) configurations.getSensorConfig(sourceType).getOrDefault(IndexingConfigurations.OUTPUT_PATH_FUNCTION_CONF, ""), message);
            SourceHandler handler = getSourceHandler(sourceType, path, configurations);
            handler.handle(message, sourceType, configurations, syncPolicyCreator);
        }
    } catch (Exception e) {
        response.addAllErrors(e, tuples);
    }
    response.addAllSuccesses(tuples);
    return response;
}

Example 5 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class SimpleHbaseEnrichmentWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
    HTableInterface table = getTable(sensorConfig);
    KeyTransformer transformer = getTransformer(sensorConfig);
    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
    String enrichmentType = enrichmentTypeObj == null ? null : enrichmentTypeObj.toString();
    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
    List<Put> puts = new ArrayList<>();
    for (JSONObject message : messages) {
        EnrichmentKey key = getKey(message, transformer, enrichmentType);
        EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
        if (key == null || value == null) {
            continue;
        }
        Put put = converter.toPut(this.cf, key, value);
        if (put != null) {
            LOG.debug("Put: {Column Family: '{}', Key: '{}', Value: '{}'}", this.cf, key, value);
            puts.add(put);
        }
    }
    BulkWriterResponse response = new BulkWriterResponse();
    try {
        table.put(puts);
    } catch (Exception e) {
        response.addAllErrors(e, tuples);
        return response;
    }
    // Can return no errors, because put will throw Exception on error.
    response.addAllSuccesses(tuples);
    return response;
}