Examples with MapVariableResolver org.apache.metron.stellar.dsl.MapVariableResolver used on opensource projects

Example 1 with MapVariableResolver

use of org.apache.metron.stellar.dsl.MapVariableResolver in project metron by apache.

the class TransformFilterExtractorDecorator method updateLookupKV.

/**
 * Returns true if lookupkv is not null after transforms and filtering on the value and indicator key
 * @param lkv LookupKV to transform and filter
 * @return true if lkv is not null after transform/filter
 */
private boolean updateLookupKV(LookupKV lkv, AtomicReference<Object> state) {
    Map<String, Object> ret = lkv.getValue().getMetadata();
    Map<String, Object> ind = new LinkedHashMap<>();
    String indicator = lkv.getKey().getIndicator();
    // add indicator as a resolvable variable. Also enable using resolved/transformed variables and values from operating on the value metadata
    ind.put(INDICATOR.toString(), indicator);
    Map<String, Object> stateMap = new LinkedHashMap<>();
    stateMap.put(STATE_KEY, state.get());
    MapVariableResolver resolver = new MapVariableResolver(ret, ind, globalConfig, stateMap);
    transform(valueTransforms, ret, resolver);
    transform(indicatorTransforms, ind, resolver);
    // update indicator
    Object updatedIndicator = ind.get(INDICATOR.toString());
    if (updatedIndicator != null || getCapabilities().contains(ExtractorCapabilities.STATEFUL)) {
        if (!(updatedIndicator instanceof String)) {
            throw new UnsupportedOperationException("Indicator transform must return String type");
        }
        lkv.getKey().setIndicator((String) updatedIndicator);
        boolean update = filter(indicatorFilter, resolver) && filter(valueFilter, resolver);
        if (update && !stateUpdate.isEmpty()) {
            transform(stateUpdate, stateMap, resolver);
            state.set(stateMap.get(STATE_KEY));
        }
        return update;
    } else {
        return false;
    }
}

Example 2 with MapVariableResolver

use of org.apache.metron.stellar.dsl.MapVariableResolver in project metron by apache.

the class StellarAdapter method enrich.

@Override
public JSONObject enrich(CacheKey value) {
    Context stellarContext = (Context) value.getConfig().getConfiguration().get(STELLAR_CONTEXT_CONF);
    ConfigHandler handler = getHandler.apply(value.getConfig());
    Map<String, Object> globalConfig = value.getConfig().getConfiguration();
    Map<String, Object> sensorConfig = value.getConfig().getEnrichment().getConfig();
    if (handler == null) {
        _LOG.trace("Stellar ConfigHandler is null.");
        return new JSONObject();
    }
    Long slowLogThreshold = null;
    if (_PERF_LOG.isDebugEnabled()) {
        slowLogThreshold = ConversionUtils.convert(globalConfig.getOrDefault(STELLAR_SLOW_LOG, STELLAR_SLOW_LOG_DEFAULT), Long.class);
    }
    // Ensure that you clone the message, because process will modify the message.  If the message object is modified
    // then cache misses will happen because the cache will be modified.
    Map<String, Object> message = new HashMap<>(value.getValue(Map.class));
    VariableResolver resolver = new MapVariableResolver(message, sensorConfig, globalConfig);
    StellarProcessor processor = new StellarProcessor();
    JSONObject enriched = process(message, handler, value.getField(), slowLogThreshold, processor, resolver, stellarContext);
    _LOG.trace("Stellar Enrichment Success: {}", enriched);
    return enriched;
}

Example 3 with MapVariableResolver

use of org.apache.metron.stellar.dsl.MapVariableResolver in project metron by apache.

the class HdfsWriter method getHdfsPathExtension.

public String getHdfsPathExtension(String sourceType, String stellarFunction, JSONObject message) {
    // If no function is provided, just use the sourceType directly
    if (stellarFunction == null || stellarFunction.trim().isEmpty()) {
        return sourceType;
    }
    // processor is a StellarProcessor();
    VariableResolver resolver = new MapVariableResolver(message);
    Object objResult = stellarProcessor.parse(stellarFunction, resolver, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
    if (objResult != null && !(objResult instanceof String)) {
        throw new IllegalArgumentException("Stellar Function <" + stellarFunction + "> did not return a String value. Returned: " + objResult);
    }
    return objResult == null ? "" : (String) objResult;
}

Example 4 with MapVariableResolver

use of org.apache.metron.stellar.dsl.MapVariableResolver in project metron by apache.

the class DefaultStellarStatefulExecutor method execute.

/**
 * Execute a Stellar expression.
 *
 * @param expression     The expression to execute.
 * @param transientState Additional state available to the expression.  This most often represents
 *                       the values available to the expression from an individual message. The state
 *                       maps a variable name to a variable's value.
 */
private Object execute(String expression, Map<String, Object> transientState) {
    VariableResolver variableResolver = new MapVariableResolver(state, transientState);
    StellarProcessor processor = new StellarProcessor();
    return processor.parse(expression, variableResolver, functionResolver, context);
}

Example 5 with MapVariableResolver

use of org.apache.metron.stellar.dsl.MapVariableResolver in project metron by apache.

the class StellarMicrobenchmark method main.

public static void main(String... argv) throws IOException {
    CommandLine cli = BenchmarkOptions.parse(new PosixParser(), argv);
    if (!BenchmarkOptions.EXPRESSIONS.has(cli)) {
        throw new IllegalStateException("You must at least specify an expressions file.");
    }
    File expressionsFile = new File(BenchmarkOptions.EXPRESSIONS.get(cli));
    Optional<File> variablesFile = Optional.ofNullable(!BenchmarkOptions.VARIABLES.has(cli) ? null : new File(BenchmarkOptions.VARIABLES.get(cli)));
    Optional<File> output = Optional.ofNullable(!BenchmarkOptions.OUTPUT.has(cli) ? null : new File(BenchmarkOptions.OUTPUT.get(cli)));
    List<String> lines = Files.readLines(expressionsFile, Charset.defaultCharset());
    Map<String, Object> variables = new HashMap<>();
    if (variablesFile.isPresent()) {
        variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), JSONUtils.MAP_SUPPLIER);
    }
    int numTimes = DEFAULT_NUM_TIMES;
    if (BenchmarkOptions.NUM_TIMES.has(cli)) {
        numTimes = Integer.parseInt(BenchmarkOptions.NUM_TIMES.get(cli));
    }
    int warmup = DEFAULT_WARMUP;
    if (BenchmarkOptions.WARMUP.has(cli)) {
        warmup = Integer.parseInt(BenchmarkOptions.WARMUP.get(cli));
    }
    Double[] percentiles = DEFAULT_PERCENTILES;
    if (BenchmarkOptions.PERCENTILES.has(cli)) {
        List<Double> percentileList = new ArrayList<>();
        for (String token : Splitter.on(",").split(BenchmarkOptions.PERCENTILES.get(cli))) {
            if (token.trim().isEmpty()) {
                continue;
            }
            Double d = Double.parseDouble(token.trim());
            percentileList.add(d);
        }
        percentiles = (Double[]) percentileList.toArray();
    }
    PrintWriter out = new PrintWriter(System.out);
    if (output.isPresent()) {
        out = new PrintWriter(output.get());
    }
    for (String statement : lines) {
        if (statement.trim().startsWith("#") || statement.trim().isEmpty()) {
            continue;
        }
        Microbenchmark.StellarStatement s = new Microbenchmark.StellarStatement();
        s.context = Context.EMPTY_CONTEXT();
        s.expression = statement;
        s.functionResolver = StellarFunctions.FUNCTION_RESOLVER();
        s.variableResolver = new MapVariableResolver(variables);
        DescriptiveStatistics stats = Microbenchmark.run(s, warmup, numTimes);
        out.println("Expression: " + statement);
        out.println(Microbenchmark.describe(stats, percentiles));
    }
    if (argv.length > 2) {
        out.close();
    }
}