Examples with StellarProcessor org.apache.metron.stellar.common.StellarProcessor used on opensource projects

Example 16 with StellarProcessor

use of org.apache.metron.stellar.common.StellarProcessor in project metron by apache.

the class StellarStatisticsFunctionsTest method run.

/**
 * Runs a Stellar expression.
 * @param expr The expression to run.
 * @param variables The variables available to the expression.
 */
private static Object run(String expr, Map<String, Object> variables) {
    StellarProcessor processor = new StellarProcessor();
    Object ret = processor.parse(expr, new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
    byte[] raw = SerDeUtils.toBytes(ret);
    Object actual = SerDeUtils.fromBytes(raw, Object.class);
    if (ret instanceof StatisticsProvider) {
        StatisticsProvider left = (StatisticsProvider) ret;
        StatisticsProvider right = (StatisticsProvider) actual;
        // N
        tolerantAssertEquals(prov -> prov.getCount(), left, right);
        // sum
        tolerantAssertEquals(prov -> prov.getSum(), left, right, 1e-3);
        // sum of squares
        tolerantAssertEquals(prov -> prov.getSumSquares(), left, right, 1e-3);
        // sum of squares
        tolerantAssertEquals(prov -> prov.getSumLogs(), left, right, 1e-3);
        // Mean
        tolerantAssertEquals(prov -> prov.getMean(), left, right, 1e-3);
        // Quadratic Mean
        tolerantAssertEquals(prov -> prov.getQuadraticMean(), left, right, 1e-3);
        // SD
        tolerantAssertEquals(prov -> prov.getStandardDeviation(), left, right, 1e-3);
        // Variance
        tolerantAssertEquals(prov -> prov.getVariance(), left, right, 1e-3);
        // Min
        tolerantAssertEquals(prov -> prov.getMin(), left, right, 1e-3);
        // Max
        tolerantAssertEquals(prov -> prov.getMax(), left, right, 1e-3);
        // Kurtosis
        tolerantAssertEquals(prov -> prov.getKurtosis(), left, right, 1e-3);
        // Skewness
        tolerantAssertEquals(prov -> prov.getSkewness(), left, right, 1e-3);
        for (double d = 10.0; d < 100.0; d += 10) {
            final double pctile = d;
            // This is a sketch, so we're a bit more forgiving here in our choice of \epsilon.
            tolerantAssertEquals(prov -> prov.getPercentile(pctile), left, right, 1e-2);
        }
    }
    return ret;
}

Example 17 with StellarProcessor

use of org.apache.metron.stellar.common.StellarProcessor in project metron by apache.

the class ThreatTriageConfig method setRiskLevelRules.

public void setRiskLevelRules(List<RiskLevelRule> riskLevelRules) {
    List<RiskLevelRule> rules = new ArrayList<>();
    Set<String> ruleIndex = new HashSet<>();
    StellarPredicateProcessor predicateProcessor = new StellarPredicateProcessor();
    StellarProcessor processor = new StellarProcessor();
    for (RiskLevelRule rule : riskLevelRules) {
        if (rule.getRule() == null || rule.getScore() == null) {
            throw new IllegalStateException("Risk level rules must contain both a rule and a score.");
        }
        if (ruleIndex.contains(rule.getRule())) {
            continue;
        } else {
            ruleIndex.add(rule.getRule());
        }
        // validate the fields which are expected to be valid Stellar expressions
        predicateProcessor.validate(rule.getRule());
        if (rule.getReason() != null) {
            processor.validate(rule.getReason());
        }
        rules.add(rule);
    }
    this.riskLevelRules = rules;
}

Example 18 with StellarProcessor

use of org.apache.metron.stellar.common.StellarProcessor in project metron by apache.

the class StellarTransformation method map.

@Override
public Map<String, Object> map(Map<String, Object> input, List<String> outputField, LinkedHashMap<String, Object> fieldMappingConfig, Context context, Map<String, Object>... sensorConfig) {
    Map<String, Object> ret = new HashMap<>();
    Map<String, Object> intermediateVariables = new HashMap<>();
    Set<String> outputs = new HashSet<>(outputField);
    MapVariableResolver resolver = new MapVariableResolver(ret, intermediateVariables, input);
    resolver.add(sensorConfig);
    StellarProcessor processor = new StellarProcessor();
    for (Map.Entry<String, Object> kv : fieldMappingConfig.entrySet()) {
        String oField = kv.getKey();
        Object transformObj = kv.getValue();
        if (transformObj != null) {
            try {
                Object o = processor.parse(transformObj.toString(), resolver, StellarFunctions.FUNCTION_RESOLVER(), context);
                if (o != null) {
                    if (outputs.contains(oField)) {
                        ret.put(oField, o);
                    } else {
                        intermediateVariables.put(oField, o);
                    }
                } else {
                    if (outputs.contains(oField)) {
                        ret.put(oField, o);
                    }
                    if (o != null) {
                        intermediateVariables.put(oField, o);
                    } else {
                        // remove here, in case there are other statements
                        intermediateVariables.remove(oField);
                    }
                }
            } catch (Exception ex) {
                throw new IllegalStateException("Unable to process transformation: " + transformObj.toString() + " for " + oField + " because " + ex.getMessage(), ex);
            }
        }
    }
    return ret;
}

Example 19 with StellarProcessor

use of org.apache.metron.stellar.common.StellarProcessor in project metron by apache.

the class ThreatTriageProcessor method apply.

@Nullable
@Override
public ThreatScore apply(@Nullable Map input) {
    ThreatScore threatScore = new ThreatScore();
    StellarPredicateProcessor predicateProcessor = new StellarPredicateProcessor();
    StellarProcessor processor = new StellarProcessor();
    VariableResolver resolver = new MapVariableResolver(input, sensorConfig.getConfiguration(), threatIntelConfig.getConfig());
    // attempt to apply each rule to the threat
    for (RiskLevelRule rule : threatTriageConfig.getRiskLevelRules()) {
        if (predicateProcessor.parse(rule.getRule(), resolver, functionResolver, context)) {
            // add the rule's score to the overall threat score
            String reason = execute(rule.getReason(), processor, resolver, String.class);
            RuleScore score = new RuleScore(rule, reason);
            threatScore.addRuleScore(score);
        }
    }
    // calculate the aggregate threat score
    Aggregators aggregators = threatTriageConfig.getAggregator();
    List<Number> allScores = threatScore.getRuleScores().stream().map(score -> score.getRule().getScore()).collect(Collectors.toList());
    Double aggregateScore = aggregators.aggregate(allScores, threatTriageConfig.getAggregationConfig());
    threatScore.setScore(aggregateScore);
    return threatScore;
}

Example 20 with StellarProcessor

use of org.apache.metron.stellar.common.StellarProcessor in project metron by apache.

the class DefaultStellarShellExecutor method executeStellar.

/**
 * Executes Stellar expressions.
 * @param expression The expression to execute.
 */
private StellarResult executeStellar(String expression) {
    StellarResult result;
    try {
        // execute the stellar expression
        VariableResolver variableResolver = new MapVariableResolver(getVariables());
        Object exprResult = new StellarProcessor().parse(expression, variableResolver, functionResolver, context);
        result = success(exprResult);
    } catch (Throwable t) {
        result = error(t);
    }
    return result;
}