Examples with FunctionRegistry org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry used on opensource projects

Example 1 with FunctionRegistry

use of org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry in project graylog2-server by Graylog2.

the class FunctionsSnippetsTest method registerFunctions.

@BeforeClass
@SuppressForbidden("Allow using default thread factory")
public static void registerFunctions() {
    final Map<String, Function<?>> functions = commonFunctions();
    functions.put(BooleanConversion.NAME, new BooleanConversion());
    functions.put(DoubleConversion.NAME, new DoubleConversion());
    functions.put(LongConversion.NAME, new LongConversion());
    functions.put(StringConversion.NAME, new StringConversion());
    functions.put(MapConversion.NAME, new MapConversion());
    // message related functions
    functions.put(HasField.NAME, new HasField());
    functions.put(SetField.NAME, new SetField());
    functions.put(SetFields.NAME, new SetFields());
    functions.put(RenameField.NAME, new RenameField());
    functions.put(RemoveField.NAME, new RemoveField());
    functions.put(DropMessage.NAME, new DropMessage());
    functions.put(CreateMessage.NAME, new CreateMessage());
    functions.put(CloneMessage.NAME, new CloneMessage());
    functions.put(TrafficAccountingSize.NAME, new TrafficAccountingSize());
    // route to stream mocks
    final StreamService streamService = mock(StreamService.class);
    otherStream = mock(Stream.class, "some stream id2");
    when(otherStream.isPaused()).thenReturn(false);
    when(otherStream.getTitle()).thenReturn("some name");
    when(otherStream.getId()).thenReturn("id2");
    when(streamService.loadAll()).thenReturn(Lists.newArrayList(defaultStream, otherStream));
    when(streamService.loadAllEnabled()).thenReturn(Lists.newArrayList(defaultStream, otherStream));
    streamCacheService = new StreamCacheService(eventBus, streamService, null);
    streamCacheService.startAsync().awaitRunning();
    final Provider<Stream> defaultStreamProvider = () -> defaultStream;
    functions.put(RouteToStream.NAME, new RouteToStream(streamCacheService, defaultStreamProvider));
    functions.put(RemoveFromStream.NAME, new RemoveFromStream(streamCacheService, defaultStreamProvider));
    lookupTableService = mock(LookupTableService.class, RETURNS_DEEP_STUBS);
    lookupTable = spy(LookupTable.class);
    when(lookupTableService.getTable(anyString())).thenReturn(lookupTable);
    lookupServiceFunction = new LookupTableService.Function(lookupTableService, "table");
    when(lookupTableService.newBuilder().lookupTable(anyString()).build()).thenReturn(lookupServiceFunction);
    // input related functions
    // TODO needs mock
    // functions.put(FromInput.NAME, new FromInput());
    // generic functions
    functions.put(RegexMatch.NAME, new RegexMatch());
    functions.put(RegexReplace.NAME, new RegexReplace());
    // string functions
    functions.put(Abbreviate.NAME, new Abbreviate());
    functions.put(Capitalize.NAME, new Capitalize());
    functions.put(Concat.NAME, new Concat());
    functions.put(Contains.NAME, new Contains());
    functions.put(EndsWith.NAME, new EndsWith());
    functions.put(Lowercase.NAME, new Lowercase());
    functions.put(Substring.NAME, new Substring());
    functions.put(Swapcase.NAME, new Swapcase());
    functions.put(Uncapitalize.NAME, new Uncapitalize());
    functions.put(Uppercase.NAME, new Uppercase());
    functions.put(KeyValue.NAME, new KeyValue());
    functions.put(Join.NAME, new Join());
    functions.put(Split.NAME, new Split());
    functions.put(StartsWith.NAME, new StartsWith());
    functions.put(Replace.NAME, new Replace());
    functions.put(Length.NAME, new Length());
    functions.put(FirstNonNull.NAME, new FirstNonNull());
    final ObjectMapper objectMapper = new ObjectMapperProvider().get();
    functions.put(JsonParse.NAME, new JsonParse(objectMapper));
    functions.put(SelectJsonPath.NAME, new SelectJsonPath(objectMapper));
    functions.put(DateConversion.NAME, new DateConversion());
    functions.put(Now.NAME, new Now());
    functions.put(FlexParseDate.NAME, new FlexParseDate());
    functions.put(ParseDate.NAME, new ParseDate());
    functions.put(ParseUnixMilliseconds.NAME, new ParseUnixMilliseconds());
    functions.put(FormatDate.NAME, new FormatDate());
    functions.put(Years.NAME, new Years());
    functions.put(Months.NAME, new Months());
    functions.put(Weeks.NAME, new Weeks());
    functions.put(Days.NAME, new Days());
    functions.put(Hours.NAME, new Hours());
    functions.put(Minutes.NAME, new Minutes());
    functions.put(Seconds.NAME, new Seconds());
    functions.put(Millis.NAME, new Millis());
    functions.put(PeriodParseFunction.NAME, new PeriodParseFunction());
    functions.put(CRC32.NAME, new CRC32());
    functions.put(CRC32C.NAME, new CRC32C());
    functions.put(MD5.NAME, new MD5());
    functions.put(Murmur3_32.NAME, new Murmur3_32());
    functions.put(Murmur3_128.NAME, new Murmur3_128());
    functions.put(SHA1.NAME, new SHA1());
    functions.put(SHA256.NAME, new SHA256());
    functions.put(SHA512.NAME, new SHA512());
    functions.put(Base16Encode.NAME, new Base16Encode());
    functions.put(Base16Decode.NAME, new Base16Decode());
    functions.put(Base32Encode.NAME, new Base32Encode());
    functions.put(Base32Decode.NAME, new Base32Decode());
    functions.put(Base32HumanEncode.NAME, new Base32HumanEncode());
    functions.put(Base32HumanDecode.NAME, new Base32HumanDecode());
    functions.put(Base64Encode.NAME, new Base64Encode());
    functions.put(Base64Decode.NAME, new Base64Decode());
    functions.put(Base64UrlEncode.NAME, new Base64UrlEncode());
    functions.put(Base64UrlDecode.NAME, new Base64UrlDecode());
    functions.put(IpAddressConversion.NAME, new IpAddressConversion());
    functions.put(CidrMatch.NAME, new CidrMatch());
    functions.put(IsNull.NAME, new IsNull());
    functions.put(IsNotNull.NAME, new IsNotNull());
    functions.put(SyslogPriorityConversion.NAME, new SyslogPriorityConversion());
    functions.put(SyslogPriorityToStringConversion.NAME, new SyslogPriorityToStringConversion());
    functions.put(SyslogFacilityConversion.NAME, new SyslogFacilityConversion());
    functions.put(SyslogLevelConversion.NAME, new SyslogLevelConversion());
    functions.put(UrlConversion.NAME, new UrlConversion());
    functions.put(UrlDecode.NAME, new UrlDecode());
    functions.put(UrlEncode.NAME, new UrlEncode());
    functions.put(IsBoolean.NAME, new IsBoolean());
    functions.put(IsNumber.NAME, new IsNumber());
    functions.put(IsDouble.NAME, new IsDouble());
    functions.put(IsLong.NAME, new IsLong());
    functions.put(IsString.NAME, new IsString());
    functions.put(IsCollection.NAME, new IsCollection());
    functions.put(IsList.NAME, new IsList());
    functions.put(IsMap.NAME, new IsMap());
    functions.put(IsDate.NAME, new IsDate());
    functions.put(IsPeriod.NAME, new IsPeriod());
    functions.put(IsIp.NAME, new IsIp());
    functions.put(IsJson.NAME, new IsJson());
    functions.put(IsUrl.NAME, new IsUrl());
    final GrokPatternService grokPatternService = mock(GrokPatternService.class);
    final GrokPattern greedyPattern = GrokPattern.create("GREEDY", ".*");
    Set<GrokPattern> patterns = Sets.newHashSet(greedyPattern, GrokPattern.create("GREEDY", ".*"), GrokPattern.create("BASE10NUM", "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"), GrokPattern.create("NUMBER", "(?:%{BASE10NUM:UNWANTED})"), GrokPattern.create("UNDERSCORE", "(?<test_field>test)"), GrokPattern.create("NUM", "%{BASE10NUM}"));
    when(grokPatternService.loadAll()).thenReturn(patterns);
    when(grokPatternService.loadByName("GREEDY")).thenReturn(Optional.of(greedyPattern));
    final EventBus clusterBus = new EventBus();
    final GrokPatternRegistry grokPatternRegistry = new GrokPatternRegistry(clusterBus, grokPatternService, Executors.newScheduledThreadPool(1));
    functions.put(GrokMatch.NAME, new GrokMatch(grokPatternRegistry));
    functions.put(GrokExists.NAME, new GrokExists(grokPatternRegistry));
    functions.put(MetricCounterIncrement.NAME, new MetricCounterIncrement(metricRegistry));
    functions.put(LookupSetValue.NAME, new LookupSetValue(lookupTableService));
    functions.put(LookupClearKey.NAME, new LookupClearKey(lookupTableService));
    functions.put(LookupSetStringList.NAME, new LookupSetStringList(lookupTableService));
    functions.put(LookupAddStringList.NAME, new LookupAddStringList(lookupTableService));
    functions.put(LookupRemoveStringList.NAME, new LookupRemoveStringList(lookupTableService));
    functionRegistry = new FunctionRegistry(functions);
}

Example 2 with FunctionRegistry

use of org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testMetrics.

@Test
@SuppressForbidden("Allow using default thread factory")
public void testMetrics() {
    final RuleMetricsConfigService ruleMetricsConfigService = mock(RuleMetricsConfigService.class);
    when(ruleMetricsConfigService.get()).thenReturn(RuleMetricsConfigDto.createDefault());
    final ClusterEventBus clusterEventBus = new ClusterEventBus("cluster-event-bus", Executors.newSingleThreadExecutor());
    final RuleService ruleService = new InMemoryRuleService(clusterEventBus);
    ruleService.save(RuleDao.create("abc", "title", "description", "rule \"match_all\"\n" + "when true\n" + "then\n" + "end", Tools.nowUTC(), null));
    final PipelineService pipelineService = new InMemoryPipelineService(new ClusterEventBus());
    pipelineService.save(PipelineDao.create("cde", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match all\n" + "    rule \"match_all\";\n" + "stage 1 match all\n" + "    rule \"match_all\";\n" + "end\n", Tools.nowUTC(), null));
    final PipelineStreamConnectionsService pipelineStreamConnectionsService = new InMemoryPipelineStreamConnectionsService(clusterEventBus);
    pipelineStreamConnectionsService.save(PipelineConnections.create(null, DEFAULT_STREAM_ID, Collections.singleton("cde")));
    final FunctionRegistry functionRegistry = new FunctionRegistry(Collections.emptyMap());
    final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry);
    final MetricRegistry metricRegistry = new MetricRegistry();
    final ConfigurationStateUpdater stateUpdater = new ConfigurationStateUpdater(ruleService, pipelineService, pipelineStreamConnectionsService, parser, ruleMetricsConfigService, metricRegistry, Executors.newScheduledThreadPool(1), mock(EventBus.class), (currentPipelines, streamPipelineConnections, ruleMetricsConfig) -> new PipelineInterpreter.State(currentPipelines, streamPipelineConnections, ruleMetricsConfig, new MetricRegistry(), 1, true));
    final PipelineInterpreter interpreter = new PipelineInterpreter(mock(MessageQueueAcknowledger.class), metricRegistry, stateUpdater);
    interpreter.process(messageInDefaultStream("", ""));
    final SortedMap<String, Meter> meters = metricRegistry.getMeters((name, metric) -> name.startsWith(name(Pipeline.class, "cde")) || name.startsWith(name(Rule.class, "abc")));
    assertThat(meters.keySet()).containsExactlyInAnyOrder(name(Pipeline.class, "cde", "executed"), name(Pipeline.class, "cde", "stage", "0", "executed"), name(Pipeline.class, "cde", "stage", "1", "executed"), name(Rule.class, "abc", "executed"), name(Rule.class, "abc", "cde", "0", "executed"), name(Rule.class, "abc", "cde", "1", "executed"), name(Rule.class, "abc", "matched"), name(Rule.class, "abc", "cde", "0", "matched"), name(Rule.class, "abc", "cde", "1", "matched"), name(Rule.class, "abc", "not-matched"), name(Rule.class, "abc", "cde", "0", "not-matched"), name(Rule.class, "abc", "cde", "1", "not-matched"), name(Rule.class, "abc", "failed"), name(Rule.class, "abc", "cde", "0", "failed"), name(Rule.class, "abc", "cde", "1", "failed"));
    assertThat(meters.get(name(Pipeline.class, "cde", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Pipeline.class, "cde", "stage", "0", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Pipeline.class, "cde", "stage", "1", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "executed")).getCount()).isEqualTo(2L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "matched")).getCount()).isEqualTo(2L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "matched")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "matched")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "not-matched")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "not-matched")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "not-matched")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "failed")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "failed")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "failed")).getCount()).isEqualTo(0L);
}

Example 3 with FunctionRegistry

use of org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method createPipelineInterpreter.

@SuppressForbidden("Allow using default thread factory")
private PipelineInterpreter createPipelineInterpreter(RuleService ruleService, PipelineService pipelineService, Map<String, Function<?>> functions) {
    final RuleMetricsConfigService ruleMetricsConfigService = mock(RuleMetricsConfigService.class);
    when(ruleMetricsConfigService.get()).thenReturn(RuleMetricsConfigDto.createDefault());
    final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(MongoDbPipelineStreamConnectionsService.class);
    final Set<String> pipelineIds = pipelineService.loadAll().stream().map(PipelineDao::id).collect(Collectors.toSet());
    final PipelineConnections pipelineConnections = PipelineConnections.create("p1", DEFAULT_STREAM_ID, pipelineIds);
    when(pipelineStreamConnectionsService.loadAll()).thenReturn(Collections.singleton(pipelineConnections));
    final FunctionRegistry functionRegistry = new FunctionRegistry(functions);
    final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry);
    final ConfigurationStateUpdater stateUpdater = new ConfigurationStateUpdater(ruleService, pipelineService, pipelineStreamConnectionsService, parser, ruleMetricsConfigService, new MetricRegistry(), Executors.newScheduledThreadPool(1), mock(EventBus.class), (currentPipelines, streamPipelineConnections, ruleMetricsConfig) -> new PipelineInterpreter.State(currentPipelines, streamPipelineConnections, ruleMetricsConfig, new MetricRegistry(), 1, true));
    return new PipelineInterpreter(messageQueueAcknowledger, new MetricRegistry(), stateUpdater);
}

Example 4 with FunctionRegistry

use of org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry in project graylog2-server by Graylog2.

the class PipelineServiceHelperTest method setup.

@Before
public void setup() {
    final Map<String, Function<?>> functions = ImmutableMap.of();
    final PipelineRuleParser pipelineRuleParser = new PipelineRuleParser(new FunctionRegistry(functions));
    underTest = new PipelineServiceHelper(pipelineRuleParser);
}