Examples with Result org.graylog2.plugin.inputs.Extractor.Result used on opensource projects

Example 21 with Result

use of org.graylog2.plugin.inputs.Extractor.Result in project graylog2-server by Graylog2.

the class StreamRouterEngine method match.

/**
     * Returns a list of matching streams for the given message.
     *
     * @param message the message
     * @return the list of matching streams
     */
public List<Stream> match(Message message) {
    final Set<Stream> result = Sets.newHashSet();
    final Set<Stream> blackList = Sets.newHashSet();
    for (final Rule rule : rulesList) {
        if (blackList.contains(rule.getStream())) {
            continue;
        }
        final StreamRule streamRule = rule.getStreamRule();
        final StreamRuleType streamRuleType = streamRule.getType();
        final Stream.MatchingType matchingType = rule.getMatchingType();
        if (!ruleTypesNotNeedingFieldPresence.contains(streamRuleType) && !message.hasField(streamRule.getField())) {
            if (matchingType == Stream.MatchingType.AND) {
                result.remove(rule.getStream());
                // blacklist stream because it can't match anymore
                blackList.add(rule.getStream());
            }
            continue;
        }
        final Stream stream;
        if (streamRuleType != StreamRuleType.REGEX) {
            stream = rule.match(message);
        } else {
            stream = rule.matchWithTimeOut(message, streamProcessingTimeout, TimeUnit.MILLISECONDS);
        }
        if (stream == null) {
            if (matchingType == Stream.MatchingType.AND) {
                result.remove(rule.getStream());
                // blacklist stream because it can't match anymore
                blackList.add(rule.getStream());
            }
        } else {
            result.add(stream);
            if (matchingType == Stream.MatchingType.OR) {
                // blacklist stream because it is already matched
                blackList.add(rule.getStream());
            }
        }
    }
    final Stream defaultStream = defaultStreamProvider.get();
    boolean alreadyRemovedDefaultStream = false;
    for (Stream stream : result) {
        streamMetrics.markIncomingMeter(stream.getId());
        if (stream.getRemoveMatchesFromDefaultStream()) {
            if (alreadyRemovedDefaultStream || message.removeStream(defaultStream)) {
                alreadyRemovedDefaultStream = true;
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Successfully removed default stream <{}> from message <{}>", defaultStream.getId(), message.getId());
                }
            } else {
                if (LOG.isWarnEnabled()) {
                    LOG.warn("Couldn't remove default stream <{}> from message <{}>", defaultStream.getId(), message.getId());
                }
            }
        }
    }
    // or someone removed it, in which case we don't mark it.
    if (!alreadyRemovedDefaultStream) {
        streamMetrics.markIncomingMeter(defaultStream.getId());
    }
    return ImmutableList.copyOf(result);
}

Example 22 with Result

use of org.graylog2.plugin.inputs.Extractor.Result in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategy method shouldRotate.

@Nullable
@Override
protected Result shouldRotate(String index, IndexSet indexSet) {
    final IndexSetConfig indexSetConfig = requireNonNull(indexSet.getConfig(), "Index set configuration must not be null");
    final String indexSetId = indexSetConfig.id();
    checkState(!isNullOrEmpty(index), "Index name must not be null or empty");
    checkState(!isNullOrEmpty(indexSetId), "Index set ID must not be null or empty");
    checkState(indexSetConfig.rotationStrategy() instanceof TimeBasedRotationStrategyConfig, "Invalid rotation strategy config <" + indexSetConfig.rotationStrategy().getClass().getCanonicalName() + "> for index set <" + indexSetId + ">");
    final TimeBasedRotationStrategyConfig config = (TimeBasedRotationStrategyConfig) indexSetConfig.rotationStrategy();
    final Period rotationPeriod = config.rotationPeriod().normalizedStandard();
    final DateTime now = Tools.nowUTC();
    // when first started, we might not know the last rotation time, look up the creation time of the index instead.
    if (!lastRotation.containsKey(indexSetId)) {
        final DateTime creationDate = indices.indexCreationDate(index);
        if (creationDate != null) {
            final DateTime currentAnchor = determineRotationPeriodAnchor(creationDate, rotationPeriod);
            anchor.put(indexSetId, currentAnchor);
            lastRotation.put(indexSetId, creationDate);
        }
        // still not able to figure out the last rotation time, we'll rotate forcibly
        if (!lastRotation.containsKey(indexSetId)) {
            return new SimpleResult(true, "No known previous rotation time, forcing index rotation now.");
        }
    }
    final DateTime currentAnchor = anchor.get(indexSetId);
    final DateTime nextRotation = currentAnchor.plus(rotationPeriod);
    if (nextRotation.isAfter(now)) {
        final String message = new MessageFormat("Next rotation at {0}", Locale.ENGLISH).format(new Object[] { nextRotation });
        return new SimpleResult(false, message);
    }
    // determine new anchor (push it to within less then one period before now) in case we missed one or more periods
    DateTime tmpAnchor;
    int multiplicator = 0;
    do {
        tmpAnchor = currentAnchor.withPeriodAdded(rotationPeriod, ++multiplicator);
    } while (tmpAnchor.isBefore(now));
    final DateTime nextAnchor = currentAnchor.withPeriodAdded(rotationPeriod, multiplicator - 1);
    anchor.put(indexSetId, nextAnchor);
    lastRotation.put(indexSetId, now);
    final String message = new MessageFormat("Rotation period {0} elapsed, next rotation at {1}", Locale.ENGLISH).format(new Object[] { now, nextAnchor });
    return new SimpleResult(true, message);
}

Example 23 with Result

use of org.graylog2.plugin.inputs.Extractor.Result in project graylog2-server by Graylog2.

the class Searches method scroll.

public ScrollResult scroll(String query, TimeRange range, int limit, int offset, List<String> fields, String filter) {
    final Set<String> indices = determineAffectedIndices(range, filter);
    // only request the fields we asked for otherwise we can't figure out which fields will be in the result set
    // until we've scrolled through the entire set.
    // TODO: Check if we can get away without loading the _source field.
    // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-fields.html#search-request-fields
    // "For backwards compatibility, if the fields parameter specifies fields which are not stored , it will
    // load the _source and extract it from it. This functionality has been replaced by the source filtering
    // parameter." -- So we should look at the source filtering parameter once we switched to ES 1.x.
    final SearchRequest request = standardSearchRequest(query, indices, limit, offset, range, filter, null, false).setScroll(new TimeValue(1, TimeUnit.MINUTES)).setSize(// TODO magic numbers
    500).addSort(SortBuilders.fieldSort(SortParseElement.DOC_FIELD_NAME)).addFields(fields.toArray(new String[fields.size()])).addField(// always request the _source field because otherwise we can't access non-stored values
    "_source").request();
    if (LOG.isDebugEnabled()) {
        try {
            LOG.debug("ElasticSearch scroll query: {}", XContentHelper.convertToJson(request.source(), false));
        } catch (IOException ignored) {
        }
    }
    final SearchResponse r = c.search(request).actionGet();
    recordEsMetrics(r, range);
    return new ScrollResult(c, query, request.source(), r, fields);
}

Example 24 with Result

use of org.graylog2.plugin.inputs.Extractor.Result in project graylog2-server by Graylog2.

the class QuickvaluesWidgetStrategy method compute.

@Override
public ComputationResult compute() {
    String filter = null;
    if (!isNullOrEmpty(streamId)) {
        filter = "streams:" + streamId;
    }
    final TermsResult terms = searches.terms(field, 50, query, filter, this.timeRange);
    Map<String, Object> result = Maps.newHashMap();
    result.put("terms", terms.getTerms());
    result.put("total", terms.getTotal());
    result.put("other", terms.getOther());
    result.put("missing", terms.getMissing());
    return new ComputationResult(result, terms.took().millis());
}

Example 25 with Result

use of org.graylog2.plugin.inputs.Extractor.Result in project graylog2-server by Graylog2.

the class JsonTesterResource method testJsonExtractor.

private JsonTesterResponse testJsonExtractor(String testString, boolean flatten, String listSeparator, String keySeparator, String kvSeparator, boolean replaceKeyWhitespace, String keyWhitespaceReplacement, String keyPrefix) {
    final Map<String, Object> config = ImmutableMap.<String, Object>builder().put("flatten", flatten).put("list_separator", listSeparator).put("key_separator", keySeparator).put("kv_separator", kvSeparator).put("replace_key_whitespace", replaceKeyWhitespace).put("key_whitespace_replacement", keyWhitespaceReplacement).put("key_prefix", keyPrefix).build();
    final JsonExtractor extractor;
    try {
        extractor = new JsonExtractor(new MetricRegistry(), "test", "Test", 0L, Extractor.CursorStrategy.COPY, "test", "test", config, getCurrentUser().getName(), Collections.<Converter>emptyList(), Extractor.ConditionType.NONE, "");
    } catch (Extractor.ReservedFieldException e) {
        throw new BadRequestException("Trying to overwrite a reserved message field", e);
    } catch (ConfigurationException e) {
        throw new BadRequestException("Invalid extractor configuration", e);
    }
    final Map<String, Object> result = extractor.extractJson(testString);
    return JsonTesterResponse.create(result, flatten, listSeparator, keySeparator, kvSeparator, testString);
}