Examples with BulkWriterResponse org.apache.metron.common.writer.BulkWriterResponse used on opensource projects

Example 21 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class ParserBoltTest method testFilterSuccess.

/**
 * Tests to ensure that a message that is unfiltered results in one write and an ack.
 * @throws Exception
 */
@Test
public void testFilterSuccess() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected SensorParserConfig getSensorParserConfig() {
            try {
                return SensorParserConfig.fromBytes(Bytes.toBytes(sensorParserConfig));
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    BulkWriterResponse successResponse = mock(BulkWriterResponse.class);
    when(successResponse.getSuccesses()).thenReturn(ImmutableList.of(t1));
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(successResponse);
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {

        {
            put("field1", "blah");
        }
    }))));
    parserBolt.execute(t1);
    verify(batchWriter, times(1)).write(any(), any(), any(), any());
    verify(outputCollector, times(1)).ack(t1);
}

Example 22 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class ParserBoltTest method testBatchOfFive.

@Test
public void testBatchOfFive() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    Set<Tuple> tuples = Stream.of(t1, t2, t3, t4, t5).collect(Collectors.toSet());
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllSuccesses(tuples);
    when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(tuples), any())).thenReturn(response);
    parserBolt.withMessageFilter(filter);
    writeNonBatch(outputCollector, parserBolt, t1);
    writeNonBatch(outputCollector, parserBolt, t2);
    writeNonBatch(outputCollector, parserBolt, t3);
    writeNonBatch(outputCollector, parserBolt, t4);
    parserBolt.execute(t5);
    verify(outputCollector, times(1)).ack(t1);
    verify(outputCollector, times(1)).ack(t2);
    verify(outputCollector, times(1)).ack(t3);
    verify(outputCollector, times(1)).ack(t4);
    verify(outputCollector, times(1)).ack(t5);
}

Example 23 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class ParserBoltTest method testImplicitBatchOfOne.

@Test
public void testImplicitBatchOfOne() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(t1);
    when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(Collections.singleton(t1)), any())).thenReturn(response);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(t1);
    verify(outputCollector, times(1)).ack(t1);
}

Example 24 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class WriterBoltTest method testBatchHappyPath.

@Test
public void testBatchHappyPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 5; ++i) {
        Tuple t = mock(Tuple.class);
        when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
        tuples.add(t);
    }
    WriterBolt bolt = new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any(), any());
    }
    // Ensure the batch returns the good Tuples
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(tuples);
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(tuples.get(4));
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(0)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 25 with BulkWriterResponse

use of org.apache.metron.common.writer.BulkWriterResponse in project metron by apache.

the class WriterBoltTest method testBatchErrorWriteFailure.

@Test
public void testBatchErrorWriteFailure() throws Exception {
    ParserConfigurations configurations = getConfigurations(6);
    String sensorType = "test";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    when(errorTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any(), any());
    }
    // Add both the good and error Tuples. This simulates a seemingly good Tuple that fails on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(tuples);
    writerResponse.addSuccess(goodTuple);
    writerResponse.addError(new IllegalStateException(), errorTuple);
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
    bolt.execute(goodTuple);
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}