Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 56 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SimpleHbaseEnrichmentWriterIntegrationTest method test.

@Test
public void test() throws UnableToStartException, IOException {
    final String sensorType = "dummy";
    // the input messages to parse
    final List<byte[]> inputMessages = new ArrayList<byte[]>() {

        {
            add(Bytes.toBytes("col11,col12,col13"));
            add(Bytes.toBytes("col21,col22,col23"));
            add(Bytes.toBytes("col31,col32,col33"));
        }
    };
    // setup external components; kafka, zookeeper
    MockHBaseTableProvider.addToCache(sensorType, "cf");
    final Properties topologyProperties = new Properties();
    final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties);
    final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {

        {
            add(new KafkaComponent.Topic(sensorType, 1));
        }
    });
    topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
    SensorParserConfig parserConfig = JSONUtils.INSTANCE.load(parserConfigJSON, SensorParserConfig.class);
    System.out.println("Workspace: " + System.getProperty("user.dir"));
    System.out.println("Configs path: ../" + TestConstants.SAMPLE_CONFIG_PATH);
    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties).withGlobalConfigsPath("../" + TestConstants.SAMPLE_CONFIG_PATH).withParserSensorConfig(sensorType, parserConfig);
    ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder().withSensorTypes(Collections.singletonList(sensorType)).withTopologyProperties(topologyProperties).withBrokerUrl(kafkaComponent.getBrokerList()).withOutputTopic(parserConfig.getOutputTopic()).build();
    ComponentRunner runner = new ComponentRunner.Builder().withComponent("zk", zkServerComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("org/apache/storm", parserTopologyComponent).withMillisecondsBetweenAttempts(5000).withCustomShutdownOrder(new String[] { "org/apache/storm", "config", "kafka", "zk" }).withNumRetries(10).build();
    try {
        runner.start();
        kafkaComponent.writeMessages(sensorType, inputMessages);
        ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> result = runner.process(new Processor<List<LookupKV<EnrichmentKey, EnrichmentValue>>>() {

            List<LookupKV<EnrichmentKey, EnrichmentValue>> messages = null;

            @Override
            public ReadinessState process(ComponentRunner runner) {
                MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(sensorType);
                if (table != null && table.size() == inputMessages.size()) {
                    EnrichmentConverter converter = new EnrichmentConverter();
                    messages = new ArrayList<>();
                    try {
                        for (Result r : table.getScanner(Bytes.toBytes("cf"))) {
                            messages.add(converter.fromResult(r, "cf"));
                        }
                    } catch (IOException e) {
                    }
                    return ReadinessState.READY;
                }
                return ReadinessState.NOT_READY;
            }

            @Override
            public ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> getResult() {
                ProcessorResult.Builder<List<LookupKV<EnrichmentKey, EnrichmentValue>>> builder = new ProcessorResult.Builder();
                return builder.withResult(messages).build();
            }
        });
        Set<String> validIndicators = new HashSet<>(ImmutableList.of("col12", "col22", "col32"));
        Map<String, Map<String, String>> validMetadata = new HashMap<String, Map<String, String>>() {

            {
                put("col12", new HashMap<String, String>() {

                    {
                        put("col1", "col11");
                        put("col3", "col13");
                    }
                });
                put("col22", new HashMap<String, String>() {

                    {
                        put("col1", "col21");
                        put("col3", "col23");
                    }
                });
                put("col32", new HashMap<String, String>() {

                    {
                        put("col1", "col31");
                        put("col3", "col33");
                    }
                });
            }
        };
        for (LookupKV<EnrichmentKey, EnrichmentValue> kv : result.getResult()) {
            assertTrue(validIndicators.contains(kv.getKey().indicator));
            assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy");
            assertNotNull(kv.getValue().getMetadata().get("timestamp"));
            assertNotNull(kv.getValue().getMetadata().get("original_string"));
            Map<String, String> metadata = validMetadata.get(kv.getKey().indicator);
            for (Map.Entry<String, String> x : metadata.entrySet()) {
                assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
            }
            assertEquals(metadata.size() + 4, kv.getValue().getMetadata().size());
        }
    } finally {
        if (runner != null) {
            runner.stop();
        }
    }
}

Example 57 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBoltTest method shouldThrowExceptionOnFailedWrite.

@Test
public void shouldThrowExceptionOnFailedWrite() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    doThrow(new IllegalStateException("write failed")).when(writerHandler).write(any(), any(), any());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    JSONObject message = new JSONObject();
    message.put(Constants.GUID, "messageId");
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(new IllegalStateException("write failed")).withSensorType(Collections.singleton("yaf")).addRawMessage("originalMessage".getBytes(StandardCharsets.UTF_8));
    parserBolt.execute(t1);
    verify(bulkWriterResponseHandler, times(1)).addTupleMessageIds(t1, Collections.singletonList("messageId"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).reportError(any(IllegalStateException.class));
    verify(outputCollector, times(1)).ack(t1);
}

Example 58 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBoltTest method shouldPrepare.

@Test
public void shouldPrepare() {
    Map stormConf = mock(Map.class);
    SensorParserConfig yafConfig = mock(SensorParserConfig.class);
    when(yafConfig.getSensorTopic()).thenReturn("yafTopic");
    when(yafConfig.getParserConfig()).thenReturn(new HashMap<String, Object>() {

        {
            put(IndexingConfigurations.BATCH_SIZE_CONF, 10);
        }
    });
    ParserConfigurations parserConfigurations = mock(ParserConfigurations.class);
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        protected SensorParserConfig getSensorParserConfig(String sensorType) {
            if ("yaf".equals(sensorType)) {
                return yafConfig;
            }
            return null;
        }

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    doReturn(stellarContext).when(parserBolt).initializeStellar();
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(stormConf, topologyContext, outputCollector);
    verify(parserRunner, times(1)).init(any(Supplier.class), eq(stellarContext));
    verify(yafConfig, times(1)).init();
    Map<String, String> topicToSensorMap = parserBolt.getTopicToSensorMap();
    assertEquals(1, topicToSensorMap.size());
    assertEquals("yaf", topicToSensorMap.get("yafTopic"));
    verify(writerHandler).init(eq(stormConf), eq(topologyContext), eq(outputCollector), eq(parserConfigurations), any(AckTuplesPolicy.class), eq(14));
}

Example 59 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccessWithMultipleMessages.

@Test
public void shouldExecuteOnSuccessWithMultipleMessages() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    List<BulkMessage<JSONObject>> messages = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        String messageId = String.format("messageId%s", i + 1);
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("field", String.format("value%s", i + 1));
        messages.add(new BulkMessage<>(messageId, message));
    }
    mockParserRunner.setMessages(messages.stream().map(BulkMessage::getMessage).collect(Collectors.toList()));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        // Verify the correct message is written and ack is handled
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        InOrder inOrder = inOrder(bulkWriterResponseHandler, writerHandler);
        inOrder.verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Arrays.asList("messageId1", "messageId2", "messageId3", "messageId4", "messageId5"));
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(0), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(1), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(2), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(3), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(4), parserConfigurations);
    }
    verifyNoMoreInteractions(writerHandler, bulkWriterResponseHandler, outputCollector);
}

Example 60 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilder method getSensorParserConfig.

/**
 * Fetch the parser configuration from Zookeeper.
 *
 * @param zookeeperUrl Zookeeper URL
 * @param sensorTypes Types of sensor
 * @param configs
 * @return
 * @throws Exception
 */
private static Map<String, SensorParserConfig> getSensorParserConfig(String zookeeperUrl, List<String> sensorTypes, ParserConfigurations configs) throws Exception {
    Map<String, SensorParserConfig> parserConfigs = new HashMap<>();
    try (CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl)) {
        client.start();
        ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client);
        for (String sensorType : sensorTypes) {
            SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType);
            if (parserConfig == null) {
                throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." + "  Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command.");
            }
            parserConfigs.put(sensorType, parserConfig);
        }
    }
    return parserConfigs;
}