Examples with MessageGetStrategy org.apache.metron.common.message.MessageGetStrategy used on opensource projects

Example 1 with MessageGetStrategy

use of org.apache.metron.common.message.MessageGetStrategy in project metron by apache.

the class ParserBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
    SensorParserConfig sensorParserConfig = getSensorParserConfig();
    try {
        // we want to ack the tuple in the situation where we have are not doing a bulk write
        // otherwise we want to defer to the writerComponent who will ack on bulk commit.
        boolean ackTuple = !writer.handleAck();
        int numWritten = 0;
        if (sensorParserConfig != null) {
            Map<String, Object> metadata = getMetadata(tuple, sensorParserConfig.getReadMetadata());
            List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
            Optional<List<JSONObject>> messages = parser.parseOptional(originalMessage);
            for (JSONObject message : messages.orElse(Collections.emptyList())) {
                message.put(Constants.SENSOR_TYPE, getSensorType());
                if (sensorParserConfig.getMergeMetadata()) {
                    message.putAll(metadata);
                }
                for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
                    if (handler != null) {
                        if (!sensorParserConfig.getMergeMetadata()) {
                            // if we haven't merged metadata, then we need to pass them along as configuration params.
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig(), metadata);
                        } else {
                            handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig());
                        }
                    }
                }
                if (!message.containsKey(Constants.GUID)) {
                    message.put(Constants.GUID, UUID.randomUUID().toString());
                }
                if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) {
                    numWritten++;
                    List<FieldValidator> failedValidators = getFailedValidators(message, fieldValidations);
                    if (failedValidators.size() > 0) {
                        MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(getSensorType()).addRawMessage(message);
                        Set<String> errorFields = failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream()).collect(Collectors.toSet());
                        if (!errorFields.isEmpty()) {
                            error.withErrorFields(errorFields);
                        }
                        ErrorUtils.handleError(collector, error);
                    } else {
                        writer.write(getSensorType(), tuple, message, getConfigurations(), messageGetStrategy);
                    }
                }
            }
        }
        // then we want to handle the ack ourselves.
        if (ackTuple || numWritten == 0) {
            collector.ack(tuple);
        }
    } catch (Throwable ex) {
        handleError(originalMessage, tuple, ex, collector);
    }
}

Example 2 with MessageGetStrategy

use of org.apache.metron.common.message.MessageGetStrategy in project metron by apache.

the class EnrichmentJoinBoltTest method test.

@Test
public void test() throws IOException {
    EnrichmentJoinBolt enrichmentJoinBolt = new EnrichmentJoinBolt("zookeeperUrl");
    enrichmentJoinBolt.setCuratorFramework(client);
    enrichmentJoinBolt.setZKCache(cache);
    enrichmentJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
    enrichmentJoinBolt.withMaxCacheSize(100);
    enrichmentJoinBolt.withMaxTimeRetain(10000);
    enrichmentJoinBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
    Set<String> actualStreamIds = enrichmentJoinBolt.getStreamIds(sampleMessage);
    Assert.assertEquals(joinStreamIds, actualStreamIds);
    Map<String, Tuple> streamMessageMap = new HashMap<>();
    MessageGetStrategy messageGetStrategy = mock(MessageGetStrategy.class);
    Tuple sampleTuple = mock(Tuple.class);
    when(messageGetStrategy.get(sampleTuple)).thenReturn(sampleMessage);
    Tuple enrichedTuple = mock(Tuple.class);
    when(messageGetStrategy.get(enrichedTuple)).thenReturn(enrichedMessage);
    streamMessageMap.put("message", sampleTuple);
    streamMessageMap.put("enriched", enrichedTuple);
    JSONObject joinedMessage = enrichmentJoinBolt.joinMessages(streamMessageMap, messageGetStrategy);
    removeTimingFields(joinedMessage);
    Assert.assertEquals(expectedJoinedMessage, joinedMessage);
}

Example 3 with MessageGetStrategy

use of org.apache.metron.common.message.MessageGetStrategy in project metron by apache.

the class ThreatIntelJoinBoltTest method test.

public void test(String threatTriageConfig, boolean badConfig) throws IOException {
    ThreatIntelJoinBolt threatIntelJoinBolt = new ThreatIntelJoinBolt("zookeeperUrl");
    threatIntelJoinBolt.setCuratorFramework(client);
    threatIntelJoinBolt.setZKCache(cache);
    SensorEnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(new FileInputStream(sampleSensorEnrichmentConfigPath), SensorEnrichmentConfig.class);
    boolean withThreatTriage = threatTriageConfig != null;
    if (withThreatTriage) {
        try {
            enrichmentConfig.getThreatIntel().setTriageConfig(JSONUtils.INSTANCE.load(threatTriageConfig, ThreatTriageConfig.class));
            if (badConfig) {
                Assert.fail(threatTriageConfig + "\nThis should not parse!");
            }
        } catch (JsonMappingException pe) {
            if (!badConfig) {
                throw pe;
            }
        }
    }
    threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, enrichmentConfig);
    HashMap<String, Object> globalConfig = new HashMap<>();
    String baseDir = UnitTestHelper.findDir("GeoLite");
    File geoHdfsFile = new File(new File(baseDir), "GeoIP2-City-Test.mmdb.gz");
    globalConfig.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
    threatIntelJoinBolt.getConfigurations().updateGlobalConfig(globalConfig);
    threatIntelJoinBolt.withMaxCacheSize(100);
    threatIntelJoinBolt.withMaxTimeRetain(10000);
    threatIntelJoinBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
    Map<String, Object> fieldMap = threatIntelJoinBolt.getFieldMap("incorrectSourceType");
    Assert.assertNull(fieldMap);
    fieldMap = threatIntelJoinBolt.getFieldMap(sensorType);
    Assert.assertTrue(fieldMap.containsKey("hbaseThreatIntel"));
    MessageGetStrategy messageGetStrategy = mock(MessageGetStrategy.class);
    Tuple messageTuple = mock(Tuple.class);
    when(messageGetStrategy.get(messageTuple)).thenReturn(message);
    Map<String, Tuple> streamMessageMap = new HashMap<>();
    streamMessageMap.put("message", messageTuple);
    JSONObject joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap, messageGetStrategy);
    assertFalse(joinedMessage.containsKey("is_alert"));
    when(messageGetStrategy.get(messageTuple)).thenReturn(messageWithTiming);
    joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap, messageGetStrategy);
    assertFalse(joinedMessage.containsKey("is_alert"));
    when(messageGetStrategy.get(messageTuple)).thenReturn(alertMessage);
    joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap, messageGetStrategy);
    assertTrue(joinedMessage.containsKey("is_alert") && "true".equals(joinedMessage.get("is_alert")));
    if (withThreatTriage && !badConfig) {
        assertTrue(joinedMessage.containsKey("threat.triage.score"));
        Double score = (Double) joinedMessage.get("threat.triage.score");
        assertTrue(Math.abs(10d - score) < 1e-10);
    } else {
        assertFalse(joinedMessage.containsKey("threat.triage.score"));
    }
}