Examples with Pipeline org.graylog.plugins.pipelineprocessor.ast.Pipeline used on opensource projects

Example 6 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineFacade method resolveNativeEntity.

@Override
public Graph<EntityDescriptor> resolveNativeEntity(EntityDescriptor entityDescriptor) {
    final MutableGraph<EntityDescriptor> mutableGraph = GraphBuilder.directed().build();
    mutableGraph.addNode(entityDescriptor);
    final ModelId modelId = entityDescriptor.id();
    try {
        final PipelineDao pipelineDao = pipelineService.load(modelId.id());
        final String pipelineSource = pipelineDao.source();
        final Collection<String> referencedRules = referencedRules(pipelineSource);
        referencedRules.stream().map(ModelId::of).map(id -> EntityDescriptor.create(id, ModelTypes.PIPELINE_RULE_V1)).forEach(rule -> mutableGraph.putEdge(entityDescriptor, rule));
        final Set<PipelineConnections> pipelineConnections = connectionsService.loadByPipelineId(pipelineDao.id());
        pipelineConnections.stream().map(PipelineConnections::streamId).map(ModelId::of).map(id -> EntityDescriptor.create(id, ModelTypes.STREAM_V1)).forEach(stream -> mutableGraph.putEdge(entityDescriptor, stream));
    } catch (NotFoundException e) {
        LOG.debug("Couldn't find pipeline {}", entityDescriptor, e);
    }
    return ImmutableGraph.copyOf(mutableGraph);
}

Example 7 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineFacade method decode.

private NativeEntity<PipelineDao> decode(EntityV1 entity, Map<String, ValueReference> parameters, Map<EntityDescriptor, Object> nativeEntities) {
    final DateTime now = Tools.nowUTC();
    final PipelineEntity pipelineEntity = objectMapper.convertValue(entity.data(), PipelineEntity.class);
    final ValueReference description = pipelineEntity.description();
    final PipelineDao pipelineDao = PipelineDao.builder().title(pipelineEntity.title().asString(parameters)).description(description == null ? null : description.asString(parameters)).source(pipelineEntity.source().asString(parameters)).createdAt(now).modifiedAt(now).build();
    final PipelineDao savedPipelineDao = pipelineService.save(pipelineDao);
    final String pipelineId = requireNonNull(savedPipelineDao.id(), "Saved pipeline ID must not be null");
    final Set<EntityDescriptor> connectedStreamEntities = pipelineEntity.connectedStreams().stream().map(valueReference -> valueReference.asString(parameters)).map(streamId -> EntityDescriptor.create(streamId, ModelTypes.STREAM_V1)).collect(Collectors.toSet());
    final Set<Stream> connectedStreams = connectedStreams(connectedStreamEntities, nativeEntities);
    createPipelineConnections(pipelineId, connectedStreams);
    return NativeEntity.create(entity.id(), pipelineId, TYPE_V1, savedPipelineDao.title(), savedPipelineDao);
}

Example 8 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineFacadeTest method resolve.

@Test
@MongoDBFixtures("PipelineFacadeTest/pipelines.json")
public void resolve() {
    final Stage stage = Stage.builder().stage(0).match(Stage.Match.EITHER).ruleReferences(ImmutableList.of("debug", "no-op")).build();
    RuleDao ruleDao1 = RuleDao.builder().id("2342353045938450345").title("debug").source("rule \\\"debug\\\"\\nrule \\\"no-op\\\"\\nend\"").build();
    org.graylog.plugins.pipelineprocessor.ast.Rule rule1 = org.graylog.plugins.pipelineprocessor.ast.Rule.builder().id("1").name("debug").when(mock(LogicalExpression.class)).then(Collections.emptyList()).build();
    RuleDao ruleDao2 = RuleDao.builder().id("2342353045938450346").title("no-op").source("rule \\\"debug\\\"\\nrule \\\"no-op\\\"\\nend\"").build();
    org.graylog.plugins.pipelineprocessor.ast.Rule rule2 = org.graylog.plugins.pipelineprocessor.ast.Rule.builder().id("2").name("no-op").when(mock(LogicalExpression.class)).then(Collections.emptyList()).build();
    stage.setRules(ImmutableList.of(rule1, rule2));
    final Pipeline pipeline = Pipeline.builder().id("5a85c4854b900afd5d662be3").name("Test").stages(ImmutableSortedSet.of(stage)).build();
    when(pipelineRuleParser.parsePipeline(eq("dummy"), anyString())).thenReturn(pipeline);
    when(ruleService.findByName("no-op")).thenReturn(Optional.of(ruleDao1));
    when(ruleService.findByName("debug")).thenReturn(Optional.of(ruleDao2));
    final EntityDescriptor pipelineEntity = EntityDescriptor.create("5a85c4854b900afd5d662be3", ModelTypes.PIPELINE_V1);
    final Graph<EntityDescriptor> graph = facade.resolveNativeEntity(pipelineEntity);
    final EntityDescriptor streamEntity = EntityDescriptor.create("5adf23894b900a0fdb4e517d", ModelTypes.STREAM_V1);
    final EntityDescriptor ruleEntity1 = EntityDescriptor.create("2342353045938450345", ModelTypes.PIPELINE_RULE_V1);
    final EntityDescriptor ruleEntity2 = EntityDescriptor.create("2342353045938450346", ModelTypes.PIPELINE_RULE_V1);
    assertThat(graph.nodes()).containsOnly(pipelineEntity, streamEntity, ruleEntity1, ruleEntity2);
}

Example 9 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineFacadeTest method resolveEntityDescriptor.

@Test
@MongoDBFixtures("PipelineFacadeTest/pipelines.json")
public void resolveEntityDescriptor() {
    final Stage stage = Stage.builder().stage(0).match(Stage.Match.EITHER).ruleReferences(Collections.singletonList("no-op")).build();
    final Pipeline pipeline = Pipeline.builder().id("5a85c4854b900afd5d662be3").name("Test").stages(ImmutableSortedSet.of(stage)).build();
    when(pipelineRuleParser.parsePipeline("dummy", "pipeline \"Test\"\nstage 0 match either\nrule \"debug\"\nrule \"no-op\"\nend")).thenReturn(pipeline);
    RuleDao ruleDao = RuleDao.builder().id("2342353045938450345").title("no-op").source("rule \\\"debug\\\"\\nrule \\\"no-op\\\"\\nend\"").build();
    when(ruleService.findByName("no-op")).thenReturn(Optional.of(ruleDao));
    final EntityDescriptor descriptor = EntityDescriptor.create("5a85c4854b900afd5d662be3", ModelTypes.PIPELINE_V1);
    final Graph<EntityDescriptor> graph = facade.resolveNativeEntity(descriptor);
    assertThat(graph.nodes()).containsOnly(descriptor, EntityDescriptor.create("5adf23894b900a0fdb4e517d", ModelTypes.STREAM_V1), EntityDescriptor.create("2342353045938450345", ModelTypes.PIPELINE_RULE_V1));
}

Example 10 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class ConfigurationStateUpdater method reloadAndSave.

// only the singleton instance should mutate itself, others are welcome to reload a new state, but we don't
// currently allow direct global state updates from external sources (if you need to, send an event on the bus instead)
private synchronized PipelineInterpreter.State reloadAndSave() {
    // read all rules and parse them
    Map<String, Rule> ruleNameMap = Maps.newHashMap();
    ruleService.loadAll().forEach(ruleDao -> {
        Rule rule;
        try {
            rule = pipelineRuleParser.parseRule(ruleDao.id(), ruleDao.source(), false);
        } catch (ParseException e) {
            log.warn("Ignoring non parseable rule <{}/{}> with errors <{}>", ruleDao.title(), ruleDao.id(), e.getErrors());
            rule = Rule.alwaysFalse("Failed to parse rule: " + ruleDao.id());
        }
        ruleNameMap.put(rule.name(), rule);
    });
    // read all pipelines and parse them
    ImmutableMap.Builder<String, Pipeline> pipelineIdMap = ImmutableMap.builder();
    pipelineService.loadAll().forEach(pipelineDao -> {
        Pipeline pipeline;
        try {
            pipeline = pipelineRuleParser.parsePipeline(pipelineDao.id(), pipelineDao.source());
        } catch (ParseException e) {
            pipeline = Pipeline.empty("Failed to parse pipeline" + pipelineDao.id());
        }
        // noinspection ConstantConditions
        pipelineIdMap.put(pipelineDao.id(), resolvePipeline(pipeline, ruleNameMap));
    });
    final ImmutableMap<String, Pipeline> currentPipelines = pipelineIdMap.build();
    // read all stream connections of those pipelines to allow processing messages through them
    final HashMultimap<String, Pipeline> connections = HashMultimap.create();
    for (PipelineConnections streamConnection : pipelineStreamConnectionsService.loadAll()) {
        streamConnection.pipelineIds().stream().map(currentPipelines::get).filter(Objects::nonNull).forEach(pipeline -> connections.put(streamConnection.streamId(), pipeline));
    }
    ImmutableSetMultimap<String, Pipeline> streamPipelineConnections = ImmutableSetMultimap.copyOf(connections);
    final RuleMetricsConfigDto ruleMetricsConfig = ruleMetricsConfigService.get();
    final PipelineInterpreter.State newState = stateFactory.newState(currentPipelines, streamPipelineConnections, ruleMetricsConfig);
    latestState.set(newState);
    return newState;
}