Examples with Message org.graylog2.plugin.Message used on opensource projects

Example 81 with Message

use of org.graylog2.plugin.Message in project graylog2-server by Graylog2.

the class SetFields method evaluate.

@Override
public Void evaluate(FunctionArgs args, EvaluationContext context) {
    // noinspection unchecked
    final Map<String, Object> fields = fieldsParam.required(args, context);
    final Message message = messageParam.optional(args, context).orElse(context.currentMessage());
    final Optional<String> prefix = prefixParam.optional(args, context);
    final Optional<String> suffix = suffixParam.optional(args, context);
    if (fields != null) {
        fields.forEach((field, value) -> {
            if (prefix.isPresent()) {
                field = prefix.get() + field;
            }
            if (suffix.isPresent()) {
                field = field + suffix.get();
            }
            message.addField(field, value);
        });
    }
    return null;
}

Example 82 with Message

use of org.graylog2.plugin.Message in project graylog2-server by Graylog2.

the class PipelineInterpreter method process.

/**
 * Evaluates all pipelines that apply to the given messages, based on the current stream routing
 * of the messages.
 *
 * The processing loops on each single message (passed in or created by pipelines) until the set
 * of streams does not change anymore. No cycle detection is performed.
 *
 * @param messages            the messages to process through the pipelines
 * @param interpreterListener a listener which gets called for each processing stage (e.g. to
 *                            trace execution)
 * @param state               the pipeline/stage/rule/stream connection state to use during
 *                            processing
 * @return the processed messages
 */
public Messages process(Messages messages, InterpreterListener interpreterListener, State state) {
    interpreterListener.startProcessing();
    // message id + stream id
    final Set<Tuple2<String, String>> processingBlacklist = Sets.newHashSet();
    final List<Message> toProcess = Lists.newArrayList(messages);
    final List<Message> fullyProcessed = Lists.newArrayListWithExpectedSize(toProcess.size());
    while (!toProcess.isEmpty()) {
        final MessageCollection currentSet = new MessageCollection(toProcess);
        // we'll add them back below
        toProcess.clear();
        for (Message message : currentSet) {
            final String msgId = message.getId();
            // this makes a copy of the list, which is mutated later in updateStreamBlacklist
            // it serves as a worklist, to keep track of which <msg, stream> tuples need to be re-run again
            final Set<String> initialStreamIds = message.getStreams().stream().map(Stream::getId).collect(Collectors.toSet());
            final ImmutableSet<Pipeline> pipelinesToRun = selectPipelines(interpreterListener, processingBlacklist, message, initialStreamIds, state.getStreamPipelineConnections());
            toProcess.addAll(processForResolvedPipelines(message, msgId, pipelinesToRun, interpreterListener, state));
            // add each processed message-stream combination to the blacklist set and figure out if the processing
            // has added a stream to the message, in which case we need to cycle and determine whether to process
            // its pipeline connections, too
            boolean addedStreams = updateStreamBlacklist(processingBlacklist, message, initialStreamIds);
            potentiallyDropFilteredMessage(message);
            // go to 1 and iterate over all messages again until no more streams are being assigned
            if (!addedStreams || message.getFilterOut()) {
                log.debug("[{}] no new streams matches or dropped message, not running again", msgId);
                fullyProcessed.add(message);
            } else {
                // process again, we've added a stream
                log.debug("[{}] new streams assigned, running again for those streams", msgId);
                toProcess.add(message);
            }
        }
    }
    interpreterListener.finishProcessing();
    // 7. return the processed messages
    return new MessageCollection(fullyProcessed);
}

Example 83 with Message

use of org.graylog2.plugin.Message in project graylog2-server by Graylog2.

the class ResultMessage method setMessage.

public void setMessage(String id, Map<String, Object> message) {
    Map<String, Object> tmp = Maps.newHashMap();
    tmp.putAll(message);
    tmp.put(Message.FIELD_ID, id);
    if (tmp.containsKey(Message.FIELD_TIMESTAMP)) {
        final Object tsField = tmp.get(Message.FIELD_TIMESTAMP);
        try {
            tmp.put(Message.FIELD_TIMESTAMP, ES_DATE_FORMAT_FORMATTER.parseDateTime(String.valueOf(tsField)));
        } catch (IllegalArgumentException e) {
            // could not parse date string, this is likely a bug, but we will leave the original value alone
            LOG.warn("Could not parse timestamp of message {}", message.get("id"), e);
        }
    }
    this.message = new Message(tmp);
}

Example 84 with Message

use of org.graylog2.plugin.Message in project graylog2-server by Graylog2.

the class SearchResult method extractFields.

@VisibleForTesting
Set<String> extractFields(List<ResultMessage> hits) {
    Set<String> filteredFields = Sets.newHashSet();
    hits.forEach(hit -> {
        final Message message = hit.getMessage();
        for (String field : message.getFieldNames()) {
            if (!Message.FILTERED_FIELDS.contains(field)) {
                filteredFields.add(field);
            }
        }
    });
    return filteredFields;
}

Example 85 with Message

use of org.graylog2.plugin.Message in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategy method shouldRotate.

@Nullable
@Override
protected Result shouldRotate(String index, IndexSet indexSet) {
    final IndexSetConfig indexSetConfig = requireNonNull(indexSet.getConfig(), "Index set configuration must not be null");
    final String indexSetId = indexSetConfig.id();
    checkState(!isNullOrEmpty(index), "Index name must not be null or empty");
    checkState(!isNullOrEmpty(indexSetId), "Index set ID must not be null or empty");
    checkState(indexSetConfig.rotationStrategy() instanceof TimeBasedRotationStrategyConfig, "Invalid rotation strategy config <" + indexSetConfig.rotationStrategy().getClass().getCanonicalName() + "> for index set <" + indexSetId + ">");
    final TimeBasedRotationStrategyConfig config = (TimeBasedRotationStrategyConfig) indexSetConfig.rotationStrategy();
    Period rotationPeriod = config.rotationPeriod();
    Period maxPeriod = elasticsearchConfiguration.getMaxWriteIndexAge();
    boolean overriding = false;
    if (maxPeriod != null && isLonger(rotationPeriod, maxPeriod)) {
        log.debug("Max rotation limit {} overrides configured period {}", maxPeriod, rotationPeriod);
        rotationPeriod = maxPeriod;
        overriding = true;
    }
    final Period normalizedPeriod = rotationPeriod.normalizedStandard();
    // when first started, we might not know the last rotation time, look up the creation time of the index instead.
    if (!lastRotation.containsKey(indexSetId)) {
        indices.indexCreationDate(index).ifPresent(creationDate -> {
            final DateTime currentAnchor = determineRotationPeriodAnchor(creationDate, normalizedPeriod);
            anchor.put(indexSetId, currentAnchor);
            lastRotation.put(indexSetId, creationDate);
        });
        // still not able to figure out the last rotation time, we'll rotate forcibly
        if (!lastRotation.containsKey(indexSetId)) {
            return new SimpleResult(true, "No known previous rotation time, forcing index rotation now.");
        }
    }
    final DateTime now = Tools.nowUTC();
    final DateTime currentAnchor = anchor.get(indexSetId);
    final DateTime nextRotation = currentAnchor.plus(normalizedPeriod);
    if (nextRotation.isAfter(now)) {
        final String message = new MessageFormat("Next rotation at {0} {1}", Locale.ENGLISH).format(new Object[] { nextRotation, overriding ? "(elasticsearch_max_write_index_age overrides configured period)" : "" });
        return new SimpleResult(false, message);
    }
    // determine new anchor (push it to within less then one period before now) in case we missed one or more periods
    DateTime tmpAnchor;
    int multiplicator = 0;
    do {
        tmpAnchor = currentAnchor.withPeriodAdded(normalizedPeriod, ++multiplicator);
    } while (tmpAnchor.isBefore(now));
    final DateTime nextAnchor = currentAnchor.withPeriodAdded(normalizedPeriod, multiplicator - 1);
    anchor.put(indexSetId, nextAnchor);
    lastRotation.put(indexSetId, now);
    final String message = new MessageFormat("Rotation period {0} elapsed, next rotation at {1} {2}", Locale.ENGLISH).format(new Object[] { now, nextAnchor, overriding ? "(elasticsearch_max_write_index_age overrides configured period)" : "" });
    return new SimpleResult(true, message);
}