Search in sources :

Example 1 with Reference

use of org.graylog2.contentpacks.model.entities.references.Reference in project graylog2-server by Graylog2.

the class ObjectMapperModule method configure.

@Override
protected void configure() {
    // the ObjectMapperProvider requires at least an empty JacksonSubtypes set.
    // if the multibinder wasn't created that reference will be null, so we force its creation here
    jacksonSubTypesBinder();
    install(new GRNModule());
    bind(ClassLoader.class).annotatedWith(GraylogClassLoader.class).toInstance(classLoader);
    bind(ObjectMapper.class).toProvider(ObjectMapperProvider.class).asEagerSingleton();
}
Also used : GraylogClassLoader(org.graylog2.shared.plugins.GraylogClassLoader) GRNModule(org.graylog.grn.GRNModule) ObjectMapperProvider(org.graylog2.shared.bindings.providers.ObjectMapperProvider)

Example 2 with Reference

use of org.graylog2.contentpacks.model.entities.references.Reference in project graylog2-server by Graylog2.

the class FunctionsSnippetsTest method json.

@Test
public void json() {
    final String flatJson = "{\"str\":\"foobar\",\"int\":42,\"float\":2.5,\"bool\":true,\"array\":[1,2,3]}";
    final String nestedJson = "{\n" + "    \"store\": {\n" + "        \"book\": {\n" + "            \"category\": \"reference\",\n" + "            \"author\": \"Nigel Rees\",\n" + "            \"title\": \"Sayings of the Century\",\n" + "            \"price\": 8.95\n" + "        },\n" + "        \"bicycle\": {\n" + "            \"color\": \"red\",\n" + "            \"price\": 19.95\n" + "        }\n" + "    },\n" + "    \"expensive\": 10\n" + "}";
    final Rule rule = parser.parseRule(ruleForTest(), false);
    final Message message = new Message("JSON", "test", Tools.nowUTC());
    message.addField("flat_json", flatJson);
    message.addField("nested_json", nestedJson);
    final Message evaluatedMessage = evaluateRule(rule, message);
    assertThat(evaluatedMessage.getField("message")).isEqualTo("JSON");
    assertThat(evaluatedMessage.getField("flat_json")).isEqualTo(flatJson);
    assertThat(evaluatedMessage.getField("nested_json")).isEqualTo(nestedJson);
    assertThat(evaluatedMessage.getField("str")).isEqualTo("foobar");
    assertThat(evaluatedMessage.getField("int")).isEqualTo(42);
    assertThat(evaluatedMessage.getField("float")).isEqualTo(2.5);
    assertThat(evaluatedMessage.getField("bool")).isEqualTo(true);
    assertThat(evaluatedMessage.getField("array")).isEqualTo(Arrays.asList(1, 2, 3));
    assertThat(evaluatedMessage.getField("store")).isInstanceOf(Map.class);
    assertThat(evaluatedMessage.getField("expensive")).isEqualTo(10);
}
Also used : CreateMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage) CloneMessage(org.graylog.plugins.pipelineprocessor.functions.messages.CloneMessage) DropMessage(org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage) Message(org.graylog2.plugin.Message) IsString(org.graylog.plugins.pipelineprocessor.functions.conversion.IsString) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) MockitoRule(org.mockito.junit.MockitoRule) Rule(org.graylog.plugins.pipelineprocessor.ast.Rule) BaseParserTest(org.graylog.plugins.pipelineprocessor.BaseParserTest) Test(org.junit.Test)

Example 3 with Reference

use of org.graylog2.contentpacks.model.entities.references.Reference in project graylog2-server by Graylog2.

the class IndicesAdapterES6 method move.

@Override
public void move(String source, String target, Consumer<IndexMoveResult> resultCallback) {
    // TODO: This method should use the Re-index API: https://www.elastic.co/guide/en/elasticsearch/reference/5.3/docs-reindex.html
    final String query = SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()).size(350).sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME)).toString();
    final Search request = new Search.Builder(query).setParameter(Parameters.SCROLL, "10s").addIndex(source).build();
    final SearchResult searchResult = JestUtils.execute(jestClient, request, () -> "Couldn't process search query response");
    final String scrollId = searchResult.getJsonObject().path("_scroll_id").asText(null);
    if (scrollId == null) {
        throw new ElasticsearchException("Couldn't find scroll ID in search query response");
    }
    while (true) {
        final SearchScroll scrollRequest = new SearchScroll.Builder(scrollId, "1m").build();
        final JestResult scrollResult = JestUtils.execute(jestClient, scrollRequest, () -> "Couldn't process result of scroll query");
        final JsonNode scrollHits = scrollResult.getJsonObject().path("hits").path("hits");
        // No more hits.
        if (scrollHits.size() == 0) {
            break;
        }
        final Bulk.Builder bulkRequestBuilder = new Bulk.Builder();
        for (JsonNode jsonElement : scrollHits) {
            Optional.ofNullable(jsonElement.path("_source")).map(sourceJson -> objectMapper.<Map<String, Object>>convertValue(sourceJson, TypeReferences.MAP_STRING_OBJECT)).ifPresent(doc -> {
                final String id = (String) doc.remove("_id");
                if (!Strings.isNullOrEmpty(id)) {
                    bulkRequestBuilder.addAction(indexingHelper.prepareIndexRequest(target, doc, id));
                }
            });
        }
        final BulkResult bulkResult = JestUtils.execute(jestClient, bulkRequestBuilder.build(), () -> "Couldn't bulk index messages into index " + target);
        final boolean hasFailedItems = !bulkResult.getFailedItems().isEmpty();
        final IndexMoveResult result = IndexMoveResult.create(bulkResult.getItems().size(), bulkResult.getJsonObject().path("took").asLong(), hasFailedItems);
        resultCallback.accept(result);
    }
}
Also used : TermsAggregation(io.searchbox.core.search.aggregation.TermsAggregation) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) PutTemplate(io.searchbox.indices.template.PutTemplate) LoggerFactory(org.slf4j.LoggerFactory) ModifyAliases(io.searchbox.indices.aliases.ModifyAliases) RequestConfig(org.apache.http.client.config.RequestConfig) TypeReferences(org.graylog2.jackson.TypeReferences) UpdateSettings(io.searchbox.indices.settings.UpdateSettings) FieldSortBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.FieldSortBuilder) MaxAggregation(io.searchbox.core.search.aggregation.MaxAggregation) IndicesAdapter(org.graylog2.indexer.indices.IndicesAdapter) HealthStatus(org.graylog2.indexer.indices.HealthStatus) JestUtils(org.graylog.storage.elasticsearch6.jest.JestUtils) Indices(org.graylog2.indexer.indices.Indices) Locale(java.util.Locale) Map(java.util.Map) JsonNode(com.fasterxml.jackson.databind.JsonNode) IndexRangeStats(org.graylog2.indexer.searches.IndexRangeStats) Bulk(io.searchbox.core.Bulk) Cat(io.searchbox.core.Cat) IndexMapping(org.graylog2.indexer.IndexMapping) FilterAggregation(io.searchbox.core.search.aggregation.FilterAggregation) QueryBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilders) SearchSourceBuilder.searchSource(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder.searchSource) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) Set(java.util.Set) Health(io.searchbox.cluster.Health) AddAliasMapping(io.searchbox.indices.aliases.AddAliasMapping) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) GetAliases(io.searchbox.indices.aliases.GetAliases) RemoveAliasMapping(io.searchbox.indices.aliases.RemoveAliasMapping) PutMapping(io.searchbox.indices.mapping.PutMapping) DeleteIndex(io.searchbox.indices.DeleteIndex) Stats(io.searchbox.indices.Stats) List(java.util.List) Parameters(io.searchbox.params.Parameters) OpenIndex(io.searchbox.indices.OpenIndex) AggregationBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.AggregationBuilders) IndexMoveResult(org.graylog2.indexer.indices.IndexMoveResult) Optional(java.util.Optional) UnsupportedEncodingException(java.io.UnsupportedEncodingException) SearchSourceBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder) SearchResult(io.searchbox.core.SearchResult) FilterAggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder) HashMap(java.util.HashMap) ElasticsearchException(org.graylog2.indexer.ElasticsearchException) SearchType(io.searchbox.params.SearchType) Iterators(com.google.common.collect.Iterators) JestClient(io.searchbox.client.JestClient) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) SortBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.SortBuilders) ImmutableList(com.google.common.collect.ImmutableList) Flush(io.searchbox.indices.Flush) DeleteTemplate(io.searchbox.indices.template.DeleteTemplate) IndexStatistics(org.graylog2.indexer.indices.stats.IndexStatistics) Duration(com.github.joschi.jadconfig.util.Duration) StreamSupport(java.util.stream.StreamSupport) Nonnull(javax.annotation.Nonnull) GetSettings(io.searchbox.indices.settings.GetSettings) Logger(org.slf4j.Logger) MinAggregation(io.searchbox.core.search.aggregation.MinAggregation) Iterator(java.util.Iterator) IndexSettings(org.graylog2.indexer.indices.IndexSettings) IndexNotFoundException(org.graylog2.indexer.IndexNotFoundException) SearchScroll(io.searchbox.core.SearchScroll) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Search(io.searchbox.core.Search) AliasMapping(io.searchbox.indices.aliases.AliasMapping) DateTime(org.joda.time.DateTime) ForceMerge(io.searchbox.indices.ForceMerge) GetSingleAlias(org.graylog.storage.elasticsearch6.indices.GetSingleAlias) IOException(java.io.IOException) JestResult(io.searchbox.client.JestResult) Ints(com.google.common.primitives.Ints) CreateIndex(io.searchbox.indices.CreateIndex) BulkResult(io.searchbox.core.BulkResult) Consumer(java.util.function.Consumer) URLEncoder(java.net.URLEncoder) Collectors.toList(java.util.stream.Collectors.toList) CatResult(io.searchbox.core.CatResult) SerializationFeature(com.fasterxml.jackson.databind.SerializationFeature) CloseIndex(io.searchbox.indices.CloseIndex) GetTemplate(io.searchbox.indices.template.GetTemplate) Message(org.graylog2.plugin.Message) Collections(java.util.Collections) State(io.searchbox.cluster.State) FieldSortBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.FieldSortBuilder) SearchSourceBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder) FilterAggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder) SearchResult(io.searchbox.core.SearchResult) JsonNode(com.fasterxml.jackson.databind.JsonNode) ElasticsearchException(org.graylog2.indexer.ElasticsearchException) IndexMoveResult(org.graylog2.indexer.indices.IndexMoveResult) Bulk(io.searchbox.core.Bulk) BulkResult(io.searchbox.core.BulkResult) SearchScroll(io.searchbox.core.SearchScroll) Search(io.searchbox.core.Search) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) JestResult(io.searchbox.client.JestResult)

Example 4 with Reference

use of org.graylog2.contentpacks.model.entities.references.Reference in project graylog2-server by Graylog2.

the class ReferenceConverter method convert.

@Override
public Reference convert(JsonNode jsonNode) {
    if (jsonNode.isObject()) {
        final ImmutableSet<String> fieldNames = ImmutableSet.copyOf(jsonNode.fieldNames());
        if (fieldNames.equals(EXPECTED_FIELD_NAMES)) {
            // TODO: Possible to use ValueTypeDeserializer to avoid duplication?
            final String valueTypeText = jsonNode.path(ValueReference.FIELD_TYPE).asText();
            final ValueType valueType = ValueType.valueOf(valueTypeText.toUpperCase(Locale.ROOT));
            final JsonNode value = jsonNode.path(ValueReference.FIELD_VALUE);
            if (valueType == ValueType.BOOLEAN && value.isBoolean()) {
                return ValueReference.of(value.booleanValue());
            } else if (valueType == ValueType.DOUBLE && value.isDouble()) {
                return ValueReference.of(value.doubleValue());
            } else if (valueType == ValueType.FLOAT && value.isFloat()) {
                return ValueReference.of(value.floatValue());
            } else if (valueType == ValueType.INTEGER && value.isInt()) {
                return ValueReference.of(value.intValue());
            } else if (valueType == ValueType.LONG && (value.isLong() || value.isInt())) {
                // Jackson actually creates an int value for a small number so we check for both (long and int value) here
                return ValueReference.of(value.longValue());
            } else if (valueType == ValueType.STRING && value.isTextual()) {
                return ValueReference.of(value.textValue());
            } else if (valueType == ValueType.PARAMETER && value.isTextual()) {
                return ValueReference.createParameter(value.textValue());
            } else {
                return null;
            }
        } else {
            final ImmutableMap.Builder<String, Reference> map = ImmutableMap.builder();
            final Iterator<Map.Entry<String, JsonNode>> fields = jsonNode.fields();
            while (fields.hasNext()) {
                final Map.Entry<String, JsonNode> entry = fields.next();
                map.put(entry.getKey(), convert(entry.getValue()));
            }
            return new ReferenceMap(map.build());
        }
    } else if (jsonNode.isArray()) {
        final ImmutableList.Builder<Reference> list = ImmutableList.builder();
        for (JsonNode value : jsonNode) {
            list.add(convert(value));
        }
        return new ReferenceList(list.build());
    }
    return null;
}
Also used : ValueType(org.graylog2.contentpacks.model.entities.references.ValueType) Reference(org.graylog2.contentpacks.model.entities.references.Reference) ValueReference(org.graylog2.contentpacks.model.entities.references.ValueReference) ReferenceMap(org.graylog2.contentpacks.model.entities.references.ReferenceMap) JsonNode(com.fasterxml.jackson.databind.JsonNode) ImmutableMap(com.google.common.collect.ImmutableMap) ReferenceList(org.graylog2.contentpacks.model.entities.references.ReferenceList) ImmutableMap(com.google.common.collect.ImmutableMap) Map(java.util.Map) ReferenceMap(org.graylog2.contentpacks.model.entities.references.ReferenceMap)

Example 5 with Reference

use of org.graylog2.contentpacks.model.entities.references.Reference in project graylog2-server by Graylog2.

the class ReferenceConverterTest method convertFloatValue.

@Test
public void convertFloatValue() {
    final Reference reference = createReference("float", 100f);
    assertThat(reference).isEqualTo(ValueReference.of(100f));
}
Also used : ValueReference(org.graylog2.contentpacks.model.entities.references.ValueReference) Reference(org.graylog2.contentpacks.model.entities.references.Reference) Test(org.junit.Test)

Aggregations

Test (org.junit.Test)9 Reference (org.graylog2.contentpacks.model.entities.references.Reference)8 ValueReference (org.graylog2.contentpacks.model.entities.references.ValueReference)8 Message (org.graylog2.plugin.Message)3 JsonNode (com.fasterxml.jackson.databind.JsonNode)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 Map (java.util.Map)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 SerializationFeature (com.fasterxml.jackson.databind.SerializationFeature)1 Duration (com.github.joschi.jadconfig.util.Duration)1 Strings (com.google.common.base.Strings)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterators (com.google.common.collect.Iterators)1 Ints (com.google.common.primitives.Ints)1 JestClient (io.searchbox.client.JestClient)1 JestResult (io.searchbox.client.JestResult)1 Health (io.searchbox.cluster.Health)1 State (io.searchbox.cluster.State)1 Bulk (io.searchbox.core.Bulk)1