Search in sources :

Example 1 with PipelineService

use of org.graylog.plugins.pipelineprocessor.db.PipelineService in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testMatchPassContinuesIfNoRuleMatched.

@Test
public void testMatchPassContinuesIfNoRuleMatched() {
    final RuleService ruleService = mock(MongoDbRuleService.class);
    when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR));
    final PipelineService pipelineService = mock(MongoDbPipelineService.class);
    when(pipelineService.loadAll()).thenReturn(Collections.singleton(PipelineDao.create("p1", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match pass\n" + "    rule \"false\";\n" + "stage 1 match pass\n" + "    rule \"add_foobar\";\n" + "end\n", Tools.nowUTC(), null)));
    final Map<String, Function<?>> functions = ImmutableMap.of(SetField.NAME, new SetField());
    final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions);
    final Messages processed = interpreter.process(messageInDefaultStream("message", "test"));
    final List<Message> messages = ImmutableList.copyOf(processed);
    assertThat(messages).hasSize(1);
    final Message actualMessage = messages.get(0);
    assertThat(actualMessage.getFieldAs(String.class, "foobar")).isEqualTo("covfefe");
}
Also used : Function(org.graylog.plugins.pipelineprocessor.ast.functions.Function) Messages(org.graylog2.plugin.Messages) PipelineService(org.graylog.plugins.pipelineprocessor.db.PipelineService) MongoDbPipelineService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService) InMemoryPipelineService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService) CreateMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage) DropMessage(org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage) Message(org.graylog2.plugin.Message) RuleService(org.graylog.plugins.pipelineprocessor.db.RuleService) MongoDbRuleService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService) InMemoryRuleService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService) SetField(org.graylog.plugins.pipelineprocessor.functions.messages.SetField) Test(org.junit.Test)

Example 2 with PipelineService

use of org.graylog.plugins.pipelineprocessor.db.PipelineService in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testMetrics.

@Test
@SuppressForbidden("Allow using default thread factory")
public void testMetrics() {
    final RuleMetricsConfigService ruleMetricsConfigService = mock(RuleMetricsConfigService.class);
    when(ruleMetricsConfigService.get()).thenReturn(RuleMetricsConfigDto.createDefault());
    final ClusterEventBus clusterEventBus = new ClusterEventBus("cluster-event-bus", Executors.newSingleThreadExecutor());
    final RuleService ruleService = new InMemoryRuleService(clusterEventBus);
    ruleService.save(RuleDao.create("abc", "title", "description", "rule \"match_all\"\n" + "when true\n" + "then\n" + "end", Tools.nowUTC(), null));
    final PipelineService pipelineService = new InMemoryPipelineService(new ClusterEventBus());
    pipelineService.save(PipelineDao.create("cde", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match all\n" + "    rule \"match_all\";\n" + "stage 1 match all\n" + "    rule \"match_all\";\n" + "end\n", Tools.nowUTC(), null));
    final PipelineStreamConnectionsService pipelineStreamConnectionsService = new InMemoryPipelineStreamConnectionsService(clusterEventBus);
    pipelineStreamConnectionsService.save(PipelineConnections.create(null, DEFAULT_STREAM_ID, Collections.singleton("cde")));
    final FunctionRegistry functionRegistry = new FunctionRegistry(Collections.emptyMap());
    final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry);
    final MetricRegistry metricRegistry = new MetricRegistry();
    final ConfigurationStateUpdater stateUpdater = new ConfigurationStateUpdater(ruleService, pipelineService, pipelineStreamConnectionsService, parser, ruleMetricsConfigService, metricRegistry, Executors.newScheduledThreadPool(1), mock(EventBus.class), (currentPipelines, streamPipelineConnections, ruleMetricsConfig) -> new PipelineInterpreter.State(currentPipelines, streamPipelineConnections, ruleMetricsConfig, new MetricRegistry(), 1, true));
    final PipelineInterpreter interpreter = new PipelineInterpreter(mock(MessageQueueAcknowledger.class), metricRegistry, stateUpdater);
    interpreter.process(messageInDefaultStream("", ""));
    final SortedMap<String, Meter> meters = metricRegistry.getMeters((name, metric) -> name.startsWith(name(Pipeline.class, "cde")) || name.startsWith(name(Rule.class, "abc")));
    assertThat(meters.keySet()).containsExactlyInAnyOrder(name(Pipeline.class, "cde", "executed"), name(Pipeline.class, "cde", "stage", "0", "executed"), name(Pipeline.class, "cde", "stage", "1", "executed"), name(Rule.class, "abc", "executed"), name(Rule.class, "abc", "cde", "0", "executed"), name(Rule.class, "abc", "cde", "1", "executed"), name(Rule.class, "abc", "matched"), name(Rule.class, "abc", "cde", "0", "matched"), name(Rule.class, "abc", "cde", "1", "matched"), name(Rule.class, "abc", "not-matched"), name(Rule.class, "abc", "cde", "0", "not-matched"), name(Rule.class, "abc", "cde", "1", "not-matched"), name(Rule.class, "abc", "failed"), name(Rule.class, "abc", "cde", "0", "failed"), name(Rule.class, "abc", "cde", "1", "failed"));
    assertThat(meters.get(name(Pipeline.class, "cde", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Pipeline.class, "cde", "stage", "0", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Pipeline.class, "cde", "stage", "1", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "executed")).getCount()).isEqualTo(2L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "executed")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "matched")).getCount()).isEqualTo(2L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "matched")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "matched")).getCount()).isEqualTo(1L);
    assertThat(meters.get(name(Rule.class, "abc", "not-matched")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "not-matched")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "not-matched")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "failed")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "failed")).getCount()).isEqualTo(0L);
    assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "failed")).getCount()).isEqualTo(0L);
}
Also used : InMemoryRuleService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService) PipelineStreamConnectionsService(org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService) MongoDbPipelineStreamConnectionsService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineStreamConnectionsService) InMemoryPipelineStreamConnectionsService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineStreamConnectionsService) InMemoryPipelineStreamConnectionsService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineStreamConnectionsService) MessageQueueAcknowledger(org.graylog2.shared.messageq.MessageQueueAcknowledger) Meter(com.codahale.metrics.Meter) MetricRegistry(com.codahale.metrics.MetricRegistry) PipelineRuleParser(org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser) ClusterEventBus(org.graylog2.events.ClusterEventBus) EventBus(com.google.common.eventbus.EventBus) InMemoryPipelineService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService) ClusterEventBus(org.graylog2.events.ClusterEventBus) RuleMetricsConfigService(org.graylog.plugins.pipelineprocessor.db.RuleMetricsConfigService) Pipeline(org.graylog.plugins.pipelineprocessor.ast.Pipeline) FunctionRegistry(org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry) PipelineService(org.graylog.plugins.pipelineprocessor.db.PipelineService) MongoDbPipelineService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService) InMemoryPipelineService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService) RuleService(org.graylog.plugins.pipelineprocessor.db.RuleService) MongoDbRuleService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService) InMemoryRuleService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService) Rule(org.graylog.plugins.pipelineprocessor.ast.Rule) Test(org.junit.Test) SuppressForbidden(org.graylog2.shared.SuppressForbidden)

Example 3 with PipelineService

use of org.graylog.plugins.pipelineprocessor.db.PipelineService in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testCreateMessage.

@Test
public void testCreateMessage() {
    final RuleService ruleService = mock(MongoDbRuleService.class);
    when(ruleService.loadAll()).thenReturn(Collections.singleton(RuleDao.create("abc", "title", "description", "rule \"creates message\"\n" + "when to_string($message.message) == \"original message\"\n" + "then\n" + "  create_message(\"derived message\");\n" + "end", Tools.nowUTC(), null)));
    final PipelineService pipelineService = mock(MongoDbPipelineService.class);
    when(pipelineService.loadAll()).thenReturn(Collections.singleton(PipelineDao.create("p1", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match all\n" + "    rule \"creates message\";\n" + "end\n", Tools.nowUTC(), null)));
    final Map<String, Function<?>> functions = ImmutableMap.of(CreateMessage.NAME, new CreateMessage(), StringConversion.NAME, new StringConversion());
    final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions);
    Message msg = messageInDefaultStream("original message", "test");
    final Messages processed = interpreter.process(msg);
    final Message[] messages = Iterables.toArray(processed, Message.class);
    assertEquals(2, messages.length);
}
Also used : Function(org.graylog.plugins.pipelineprocessor.ast.functions.Function) Messages(org.graylog2.plugin.Messages) PipelineService(org.graylog.plugins.pipelineprocessor.db.PipelineService) MongoDbPipelineService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService) InMemoryPipelineService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService) CreateMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage) DropMessage(org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage) Message(org.graylog2.plugin.Message) RuleService(org.graylog.plugins.pipelineprocessor.db.RuleService) MongoDbRuleService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService) InMemoryRuleService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService) CreateMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage) StringConversion(org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion) Test(org.junit.Test)

Example 4 with PipelineService

use of org.graylog.plugins.pipelineprocessor.db.PipelineService in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testMatchAllDoesNotContinueIfNotAllRulesMatched.

@Test
public void testMatchAllDoesNotContinueIfNotAllRulesMatched() {
    final RuleService ruleService = mock(MongoDbRuleService.class);
    when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR));
    final PipelineService pipelineService = mock(MongoDbPipelineService.class);
    when(pipelineService.loadAll()).thenReturn(Collections.singleton(PipelineDao.create("p1", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match all\n" + "    rule \"true\";\n" + "    rule \"false\";\n" + "stage 1 match either\n" + "    rule \"add_foobar\";\n" + "end\n", Tools.nowUTC(), null)));
    final Map<String, Function<?>> functions = ImmutableMap.of(SetField.NAME, new SetField());
    final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions);
    final Messages processed = interpreter.process(messageInDefaultStream("message", "test"));
    final List<Message> messages = ImmutableList.copyOf(processed);
    assertThat(messages).hasSize(1);
    final Message actualMessage = messages.get(0);
    assertThat(actualMessage.hasField("foobar")).isFalse();
}
Also used : Function(org.graylog.plugins.pipelineprocessor.ast.functions.Function) Messages(org.graylog2.plugin.Messages) PipelineService(org.graylog.plugins.pipelineprocessor.db.PipelineService) MongoDbPipelineService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService) InMemoryPipelineService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService) CreateMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage) DropMessage(org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage) Message(org.graylog2.plugin.Message) RuleService(org.graylog.plugins.pipelineprocessor.db.RuleService) MongoDbRuleService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService) InMemoryRuleService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService) SetField(org.graylog.plugins.pipelineprocessor.functions.messages.SetField) Test(org.junit.Test)

Example 5 with PipelineService

use of org.graylog.plugins.pipelineprocessor.db.PipelineService in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testMatchAllContinuesIfAllRulesMatched.

@Test
public void testMatchAllContinuesIfAllRulesMatched() {
    final RuleService ruleService = mock(MongoDbRuleService.class);
    when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR));
    final PipelineService pipelineService = mock(MongoDbPipelineService.class);
    when(pipelineService.loadAll()).thenReturn(Collections.singleton(PipelineDao.create("p1", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match all\n" + "    rule \"true\";\n" + "stage 1 match either\n" + "    rule \"add_foobar\";\n" + "end\n", Tools.nowUTC(), null)));
    final Map<String, Function<?>> functions = ImmutableMap.of(SetField.NAME, new SetField());
    final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions);
    final Messages processed = interpreter.process(messageInDefaultStream("message", "test"));
    final List<Message> messages = ImmutableList.copyOf(processed);
    assertThat(messages).hasSize(1);
    final Message actualMessage = messages.get(0);
    assertThat(actualMessage.getFieldAs(String.class, "foobar")).isEqualTo("covfefe");
}
Also used : Function(org.graylog.plugins.pipelineprocessor.ast.functions.Function) Messages(org.graylog2.plugin.Messages) PipelineService(org.graylog.plugins.pipelineprocessor.db.PipelineService) MongoDbPipelineService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService) InMemoryPipelineService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService) CreateMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage) DropMessage(org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage) Message(org.graylog2.plugin.Message) RuleService(org.graylog.plugins.pipelineprocessor.db.RuleService) MongoDbRuleService(org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService) InMemoryRuleService(org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService) SetField(org.graylog.plugins.pipelineprocessor.functions.messages.SetField) Test(org.junit.Test)

Aggregations

PipelineService (org.graylog.plugins.pipelineprocessor.db.PipelineService)9 RuleService (org.graylog.plugins.pipelineprocessor.db.RuleService)9 InMemoryPipelineService (org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService)9 InMemoryRuleService (org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService)9 MongoDbPipelineService (org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService)9 MongoDbRuleService (org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService)9 Test (org.junit.Test)9 Function (org.graylog.plugins.pipelineprocessor.ast.functions.Function)8 CreateMessage (org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage)8 DropMessage (org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage)8 Message (org.graylog2.plugin.Message)8 Messages (org.graylog2.plugin.Messages)8 SetField (org.graylog.plugins.pipelineprocessor.functions.messages.SetField)7 Meter (com.codahale.metrics.Meter)1 MetricRegistry (com.codahale.metrics.MetricRegistry)1 EventBus (com.google.common.eventbus.EventBus)1 Pipeline (org.graylog.plugins.pipelineprocessor.ast.Pipeline)1 Rule (org.graylog.plugins.pipelineprocessor.ast.Rule)1 PipelineStreamConnectionsService (org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService)1 RuleMetricsConfigService (org.graylog.plugins.pipelineprocessor.db.RuleMetricsConfigService)1