Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 1 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserWriterConfigurationTest method testDefaultIndex.

@Test
public void testDefaultIndex() {
    ParserWriterConfiguration config = new ParserWriterConfiguration(new ParserConfigurations());
    Assert.assertEquals("foo", config.getIndex("foo"));
}

Example 2 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserTopologyBuilder method build.

/**
 * Builds a Storm topology that parses telemetry data received from an external sensor.
 *
 * @param zookeeperUrl             Zookeeper URL
 * @param brokerUrl                Kafka Broker URL
 * @param sensorType               Type of sensor
 * @param spoutParallelismSupplier         Supplier for the parallelism hint for the spout
 * @param spoutNumTasksSupplier            Supplier for the number of tasks for the spout
 * @param parserParallelismSupplier        Supplier for the parallelism hint for the parser bolt
 * @param parserNumTasksSupplier           Supplier for the number of tasks for the parser bolt
 * @param errorWriterParallelismSupplier   Supplier for the parallelism hint for the bolt that handles errors
 * @param errorWriterNumTasksSupplier      Supplier for the number of tasks for the bolt that handles errors
 * @param kafkaSpoutConfigSupplier         Supplier for the configuration options for the kafka spout
 * @param securityProtocolSupplier         Supplier for the security protocol
 * @param outputTopic                      The output kafka topic
 * @param stormConfigSupplier              Supplier for the storm config
 * @return A Storm topology that parses telemetry data received from an external sensor
 * @throws Exception
 */
public static ParserTopology build(String zookeeperUrl, Optional<String> brokerUrl, String sensorType, ValueSupplier<Integer> spoutParallelismSupplier, ValueSupplier<Integer> spoutNumTasksSupplier, ValueSupplier<Integer> parserParallelismSupplier, ValueSupplier<Integer> parserNumTasksSupplier, ValueSupplier<Integer> errorWriterParallelismSupplier, ValueSupplier<Integer> errorWriterNumTasksSupplier, ValueSupplier<Map> kafkaSpoutConfigSupplier, ValueSupplier<String> securityProtocolSupplier, Optional<String> outputTopic, ValueSupplier<Config> stormConfigSupplier) throws Exception {
    // fetch configuration from zookeeper
    ParserConfigurations configs = new ParserConfigurations();
    SensorParserConfig parserConfig = getSensorParserConfig(zookeeperUrl, sensorType, configs);
    int spoutParallelism = spoutParallelismSupplier.get(parserConfig, Integer.class);
    int spoutNumTasks = spoutNumTasksSupplier.get(parserConfig, Integer.class);
    int parserParallelism = parserParallelismSupplier.get(parserConfig, Integer.class);
    int parserNumTasks = parserNumTasksSupplier.get(parserConfig, Integer.class);
    int errorWriterParallelism = errorWriterParallelismSupplier.get(parserConfig, Integer.class);
    int errorWriterNumTasks = errorWriterNumTasksSupplier.get(parserConfig, Integer.class);
    Map<String, Object> kafkaSpoutConfig = kafkaSpoutConfigSupplier.get(parserConfig, Map.class);
    Optional<String> securityProtocol = Optional.ofNullable(securityProtocolSupplier.get(parserConfig, String.class));
    // create the spout
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpout kafkaSpout = createKafkaSpout(zookeeperUrl, sensorType, securityProtocol, Optional.ofNullable(kafkaSpoutConfig), parserConfig);
    builder.setSpout("kafkaSpout", kafkaSpout, spoutParallelism).setNumTasks(spoutNumTasks);
    // create the parser bolt
    ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig, outputTopic);
    builder.setBolt("parserBolt", parserBolt, parserParallelism).setNumTasks(parserNumTasks).localOrShuffleGrouping("kafkaSpout");
    // create the error bolt, if needed
    if (errorWriterNumTasks > 0) {
        WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig);
        builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism).setNumTasks(errorWriterNumTasks).localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
    }
    return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));
}

Example 3 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testFieldTransformationPriorToValidation.

@Test
public void testFieldTransformationPriorToValidation() {
    String sensorType = "dummy";
    RecordingWriter recordingWriter = new RecordingWriter();
    // create a parser which acts like a basic parser but returns no timestamp field.
    BasicParser dummyParser = new BasicParser() {

        @Override
        public void init() {
        }

        @Override
        public List<JSONObject> parse(byte[] rawMessage) {
            return ImmutableList.of(new JSONObject() {

                {
                    put("data", "foo");
                    put("timestampstr", "2016-01-05 17:02:30");
                    put("original_string", "blah");
                }
            });
        }

        @Override
        public void configure(Map<String, Object> config) {
        }
    };
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, dummyParser, new WriterHandler(recordingWriter)) {

        @Override
        protected SensorParserConfig getSensorParserConfig() {
            try {
                return SensorParserConfig.fromBytes(Bytes.toBytes(csvWithFieldTransformations));
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    when(t1.getBinary(0)).thenReturn(new byte[] {});
    parserBolt.execute(t1);
    Assert.assertEquals(1, recordingWriter.getRecords().size());
    long expected = 1452013350000L;
    Assert.assertEquals(expected, recordingWriter.getRecords().get(0).get("timestamp"));
}

Example 4 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testBatchOfFiveWithError.

@Test
public void testBatchOfFiveWithError() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(t1);
    parserBolt.execute(t2);
    parserBolt.execute(t3);
    parserBolt.execute(t4);
    parserBolt.execute(t5);
    verify(outputCollector, times(1)).ack(t1);
    verify(outputCollector, times(1)).ack(t2);
    verify(outputCollector, times(1)).ack(t3);
    verify(outputCollector, times(1)).ack(t4);
    verify(outputCollector, times(1)).ack(t5);
}

Example 5 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testBatchOfOne.

@Test
public void testBatchOfOne() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(t1);
    when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(Collections.singleton(t1)), any())).thenReturn(response);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(t1);
    verify(outputCollector, times(1)).ack(t1);
}