Examples with Pipeline org.graylog.plugins.pipelineprocessor.ast.Pipeline used on opensource projects

Example 11 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineInterpreter method evaluateStage.

private void evaluateStage(Stage stage, Message message, String msgId, List<Message> result, Set<Pipeline> pipelinesToSkip, InterpreterListener interpreterListener) {
    final Pipeline pipeline = stage.getPipeline();
    if (pipelinesToSkip.contains(pipeline)) {
        log.debug("[{}] previous stage result prevents further processing of pipeline `{}`", msgId, pipeline.name());
        return;
    }
    stage.markExecution();
    interpreterListener.enterStage(stage);
    log.debug("[{}] evaluating rule conditions in stage {}: match {}", msgId, stage.stage(), stage.match());
    // TODO the message should be decorated to allow layering changes and isolate stages
    final EvaluationContext context = new EvaluationContext(message);
    // 3. iterate over all the stages in these pipelines and execute them in order
    final List<Rule> stageRules = stage.getRules();
    final List<Rule> rulesToRun = new ArrayList<>(stageRules.size());
    // If there are no rules, we can simply continue to the next stage
    boolean anyRulesMatched = stageRules.isEmpty();
    boolean allRulesMatched = true;
    for (Rule rule : stageRules) {
        try {
            final boolean ruleCondition = evaluateRuleCondition(rule, message, msgId, pipeline, context, rulesToRun, interpreterListener);
            anyRulesMatched |= ruleCondition;
            allRulesMatched &= ruleCondition;
            if (context.hasEvaluationErrors()) {
                log.warn("Error evaluating condition for rule <{}/{}> with message: {} (Error: {})", rule.name(), rule.id(), message, context.lastEvaluationError());
                break;
            }
        } catch (Exception e) {
            log.warn("Error evaluating condition for rule <{}/{}> with message: {} (Error: {})", rule.name(), rule.id(), message, e.getMessage());
            throw e;
        }
    }
    for (Rule rule : rulesToRun) {
        if (!executeRuleActions(rule, message, msgId, pipeline, context, interpreterListener)) {
            log.warn("Error evaluating action for rule <{}/{}> with message: {} (Error: {})", rule.name(), rule.id(), message, context.lastEvaluationError());
            // if any of the rules raise an error, skip the rest of the rules
            break;
        }
    }
    // stage needed to match all rule conditions to enable the next stage,
    // record that it is ok to proceed with this pipeline
    // OR
    // any rule could match, but at least one had to,
    // record that it is ok to proceed with the pipeline
    final boolean matchAllSuccess = Stage.Match.ALL == stage.match() && allRulesMatched;
    final boolean matchEitherSuccess = Stage.Match.EITHER == stage.match() && anyRulesMatched;
    final boolean matchIsPass = Stage.Match.PASS == stage.match();
    if (matchAllSuccess || matchEitherSuccess || matchIsPass) {
        interpreterListener.continuePipelineExecution(pipeline, stage);
        log.debug("[{}] stage {} for pipeline `{}` required match: {}, ok to proceed with next stage", msgId, stage.stage(), pipeline.name(), stage.match());
    } else {
        // no longer execute stages from this pipeline, the guard prevents it
        interpreterListener.stopPipelineExecution(pipeline, stage);
        log.debug("[{}] stage {} for pipeline `{}` required match: {}, NOT ok to proceed with next stage", msgId, stage.stage(), pipeline.name(), stage.match());
        pipelinesToSkip.add(pipeline);
    }
    // 4. after each complete stage run, merge the processing changes, stages are isolated from each other
    // TODO message changes become visible immediately for now
    // 4a. also add all new messages from the context to the toProcess work list
    Iterables.addAll(result, context.createdMessages());
    context.clearCreatedMessages();
    interpreterListener.exitStage(stage);
}

Example 12 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineInterpreter method processForResolvedPipelines.

// Public access is required due to use in the Illuminate processor.
public List<Message> processForResolvedPipelines(Message message, String msgId, Set<Pipeline> pipelines, InterpreterListener interpreterListener, State state) {
    final List<Message> result = new ArrayList<>();
    // record execution of pipeline in metrics
    pipelines.forEach(Pipeline::markExecution);
    final StageIterator stages = state.getStageIterator(pipelines);
    final Set<Pipeline> pipelinesToSkip = Sets.newHashSet();
    // pipeline execution ordering is not guaranteed
    while (stages.hasNext()) {
        // remaining stages.
        if (message.getFilterOut()) {
            break;
        }
        final List<Stage> stageSet = stages.next();
        for (final Stage stage : stageSet) {
            evaluateStage(stage, message, msgId, result, pipelinesToSkip, interpreterListener);
        }
    }
    // 7. return the processed messages
    return result;
}

Example 13 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineResource method update.

@ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied")
@Path("/{id}")
@PUT
@AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_UPDATE)
public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException {
    checkPermission(PipelineRestPermissions.PIPELINE_EDIT, id);
    final PipelineDao dao = pipelineService.load(id);
    final Pipeline pipeline;
    try {
        pipeline = pipelineRuleParser.parsePipeline(update.id(), update.source());
    } catch (ParseException e) {
        throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build());
    }
    final PipelineDao toSave = dao.toBuilder().title(pipeline.name()).description(update.description()).source(update.source()).modifiedAt(DateTime.now(DateTimeZone.UTC)).build();
    final PipelineDao savedPipeline = pipelineService.save(toSave);
    return PipelineSource.fromDao(pipelineRuleParser, savedPipeline);
}

Example 14 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineSource method fromDao.

public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao) {
    Set<ParseError> errors = null;
    Pipeline pipeline = null;
    try {
        pipeline = parser.parsePipeline(dao.id(), dao.source());
    } catch (ParseException e) {
        errors = e.getErrors();
    }
    final List<StageSource> stageSources = (pipeline == null) ? Collections.emptyList() : pipeline.stages().stream().map(stage -> StageSource.builder().match(stage.match()).rules(stage.ruleReferences()).stage(stage.stage()).build()).collect(Collectors.toList());
    return builder().id(dao.id()).title(dao.title()).description(dao.description()).source(dao.source()).createdAt(dao.createdAt()).modifiedAt(dao.modifiedAt()).stages(stageSources).errors(errors).build();
}

Example 15 with Pipeline

use of org.graylog.plugins.pipelineprocessor.ast.Pipeline in project graylog2-server by Graylog2.

the class PipelineRuleParser method parsePipeline.

public Pipeline parsePipeline(String id, String source) {
    final ParseContext parseContext = new ParseContext(false);
    final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext);
    final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(source));
    lexer.removeErrorListeners();
    lexer.addErrorListener(errorListener);
    final RuleLangParser parser = new RuleLangParser(new CommonTokenStream(lexer));
    parser.setErrorHandler(new DefaultErrorStrategy());
    parser.removeErrorListeners();
    parser.addErrorListener(errorListener);
    final RuleLangParser.PipelineContext pipelineContext = parser.pipeline();
    WALKER.walk(new PipelineAstBuilder(parseContext), pipelineContext);
    if (parseContext.getErrors().isEmpty()) {
        final Pipeline pipeline = parseContext.pipelines.get(0);
        return pipeline.withId(id);
    }
    throw new ParseException(parseContext.getErrors());
}