Examples with LogstashAttribute org.opensearch.dataprepper.logstash.model.LogstashAttribute used on opensource projects

Example 1 with LogstashAttribute

use of org.opensearch.dataprepper.logstash.model.LogstashAttribute in project data-prepper by opensearch-project.

the class MutateMapper method mapAttributes.

public List<PluginModel> mapAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings) {
    List<PluginModel> models = new LinkedList<>();
    List<AddEntryConfig> adds = new LinkedList<>();
    List<RenameCopyConfig> renames = new LinkedList<>();
    List<String> deletes = new LinkedList<>();
    List<RenameCopyConfig> copies = new LinkedList<>();
    List<String> uppercases = new LinkedList<>();
    for (LogstashAttribute attr : logstashAttributes) {
        final String name = attr.getAttributeName();
        if (Objects.equals(name, "add_field")) {
            ((Map<String, Object>) attr.getAttributeValue().getValue()).forEach((key, value) -> adds.add(new AddEntryConfig(NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(key), value)));
        } else if (Objects.equals(name, "rename")) {
            ((Map<String, String>) attr.getAttributeValue().getValue()).forEach((key, value) -> renames.add(new RenameCopyConfig(NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(key), NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(value))));
        } else if (Objects.equals(name, "remove_field")) {
            deletes.addAll(((List<String>) attr.getAttributeValue().getValue()).stream().map(NestedSyntaxConverter::convertNestedSyntaxToJsonPointer).collect(Collectors.toList()));
        } else if (Objects.equals(name, "copy")) {
            ((Map<String, String>) attr.getAttributeValue().getValue()).forEach((key, value) -> copies.add(new RenameCopyConfig(NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(key), NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(value))));
        } else if (Objects.equals(name, "uppercase")) {
            uppercases.addAll(((ArrayList<String>) attr.getAttributeValue().getValue()).stream().map(NestedSyntaxConverter::convertNestedSyntaxToJsonPointer).collect(Collectors.toList()));
        }
    }
    if (!renames.isEmpty()) {
        Map<String, Object> renameMap = new HashMap<>();
        renameMap.put("entries", renames);
        PluginModel renameModel = new PluginModel("rename_keys", renameMap);
        models.add(renameModel);
    }
    if (!copies.isEmpty()) {
        Map<String, Object> copyMap = new HashMap<>();
        copyMap.put("entries", copies);
        PluginModel renameModel = new PluginModel("copy_values", copyMap);
        models.add(renameModel);
    }
    if (!adds.isEmpty()) {
        Map<String, Object> addMap = new HashMap<>();
        addMap.put("entries", adds);
        PluginModel addModel = new PluginModel("add_entries", addMap);
        models.add(addModel);
    }
    if (!deletes.isEmpty()) {
        Map<String, Object> deleteMap = new HashMap<>();
        deleteMap.put("with_keys", deletes);
        PluginModel deleteModel = new PluginModel("delete_entries", deleteMap);
        models.add(deleteModel);
    }
    if (!uppercases.isEmpty()) {
        Map<String, Object> uppercaseMap = new HashMap<>();
        uppercaseMap.put("with_keys", uppercases);
        PluginModel uppercaseModel = new PluginModel("uppercase_string", uppercaseMap);
        models.add(uppercaseModel);
    }
    return models;
}

Example 2 with LogstashAttribute

use of org.opensearch.dataprepper.logstash.model.LogstashAttribute in project data-prepper by opensearch-project.

the class GrokLogstashPluginAttributesMapperTest method mapAttributes_converts_nested_syntax_in_mapped_attributes.

@Test
void mapAttributes_converts_nested_syntax_in_mapped_attributes() {
    final LogstashAttribute matchMultiKeysLogstashAttribute = prepareHashTypeMatchLogstashAttribute(Arrays.asList(Map.entry("[text][message]", "fake message regex 1"), Map.entry("other", "fake other regex")));
    final LogstashAttribute matchMessageLogstashAttribute2 = prepareArrayTypeMatchLogstashAttribute("[text][message]", "%{NUMBER} %{GREEDYDATA:[nested][field][data2]}");
    final List<LogstashAttribute> matchLogstashAttributes = Arrays.asList(matchMultiKeysLogstashAttribute, matchMessageLogstashAttribute2);
    final Map<String, Object> expectedMatchSettings = Map.of("/text/message", Arrays.asList("fake message regex 1", "%{NUMBER} %{GREEDYDATA:/nested/field/data2}"), "other", Collections.singletonList("fake other regex"));
    final String dataPrepperMatchAttribute = "match";
    final LogstashAttributesMappings mappings = mock(LogstashAttributesMappings.class);
    when(mappings.getMappedAttributeNames()).thenReturn(Collections.singletonMap(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, dataPrepperMatchAttribute));
    final List<PluginModel> actualPluginModels = createObjectUnderTest().mapAttributes(matchLogstashAttributes, mappings);
    assertThat(actualPluginModels, notNullValue());
    assertThat(actualPluginModels.size(), equalTo(1));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperMatchAttribute));
    assertThat(actualPluginModels.get(0).getPluginSettings().get(dataPrepperMatchAttribute), equalTo(expectedMatchSettings));
}

Example 3 with LogstashAttribute

use of org.opensearch.dataprepper.logstash.model.LogstashAttribute in project data-prepper by opensearch-project.

the class GrokLogstashPluginAttributesMapperTest method mapAttributes_sets_mapped_attributes_match_with_named_captures_and_pattern_definitions.

@SuppressWarnings("unchecked")
@Test
void mapAttributes_sets_mapped_attributes_match_with_named_captures_and_pattern_definitions() {
    final String testNamedCapture = "test_named_capture";
    final String testRegex = "fake_regex";
    final String handWrittenPatternName = "handwritten_pattern_name";
    final String handWrittenPatternValue = "handwritten_pattern_value";
    final LogstashAttribute matchMessageLogstashAttributeWithNamedCaptures = prepareArrayTypeMatchLogstashAttribute("message", String.format("(?<%s>%s)", testNamedCapture, testRegex));
    final LogstashAttribute patternDefinitionsLogstashAttribute = preparePatternDefinitionsLogstashAttribute(Map.of(handWrittenPatternName, handWrittenPatternValue));
    final String dataPrepperMatchAttribute = "match";
    final String dataPrepperPatternDefinitionsAttribute = "pattern_definitions";
    final LogstashAttributesMappings mappings = mock(LogstashAttributesMappings.class);
    when(mappings.getMappedAttributeNames()).thenReturn(Map.of(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, dataPrepperMatchAttribute, LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME, dataPrepperPatternDefinitionsAttribute));
    final List<PluginModel> actualPluginModels = createObjectUnderTest().mapAttributes(Arrays.asList(matchMessageLogstashAttributeWithNamedCaptures, patternDefinitionsLogstashAttribute), mappings);
    assertThat(actualPluginModels, notNullValue());
    assertThat(actualPluginModels.size(), equalTo(1));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperMatchAttribute));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperPatternDefinitionsAttribute));
    final Map<String, String> actualPatternDefinitions = (Map<String, String>) actualPluginModels.get(0).getPluginSettings().get(dataPrepperPatternDefinitionsAttribute);
    assertThat(actualPatternDefinitions, hasKey(handWrittenPatternName));
    assertThat(actualPatternDefinitions, hasValue(handWrittenPatternValue));
    assertThat(actualPatternDefinitions, hasValue(testRegex));
    final Map<String, List<String>> actualMatch = (Map<String, List<String>>) actualPluginModels.get(0).getPluginSettings().get(dataPrepperMatchAttribute);
    assertThat(actualMatch.get("message").get(0), matchesPattern(String.format("%%\\{(.*?):%s\\}", testNamedCapture)));
}

Example 4 with LogstashAttribute

use of org.opensearch.dataprepper.logstash.model.LogstashAttribute in project data-prepper by opensearch-project.

the class GrokLogstashPluginAttributesMapperTest method mapAttributes_sets_mapped_attributes_merging_multiple_match.

@Test
void mapAttributes_sets_mapped_attributes_merging_multiple_match() {
    final LogstashAttribute matchMultiKeysLogstashAttribute = prepareHashTypeMatchLogstashAttribute(Arrays.asList(Map.entry("message", "fake message regex 1"), Map.entry("other", "fake other regex")));
    final LogstashAttribute matchMessageLogstashAttribute2 = prepareArrayTypeMatchLogstashAttribute("message", "fake message regex 2");
    final List<LogstashAttribute> matchLogstashAttributes = Arrays.asList(matchMultiKeysLogstashAttribute, matchMessageLogstashAttribute2);
    final Map<String, Object> expectedMatchSettings = Map.of("message", Arrays.asList("fake message regex 1", "fake message regex 2"), "other", Collections.singletonList("fake other regex"));
    final String dataPrepperMatchAttribute = "match";
    final LogstashAttributesMappings mappings = mock(LogstashAttributesMappings.class);
    when(mappings.getMappedAttributeNames()).thenReturn(Collections.singletonMap(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, dataPrepperMatchAttribute));
    final List<PluginModel> actualPluginModels = createObjectUnderTest().mapAttributes(matchLogstashAttributes, mappings);
    assertThat(actualPluginModels, notNullValue());
    assertThat(actualPluginModels.size(), equalTo(1));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperMatchAttribute));
    assertThat(actualPluginModels.get(0).getPluginSettings().get(dataPrepperMatchAttribute), equalTo(expectedMatchSettings));
}

Example 5 with LogstashAttribute

use of org.opensearch.dataprepper.logstash.model.LogstashAttribute in project data-prepper by opensearch-project.

the class GrokLogstashPluginAttributesMapperTest method preparePatternDefinitionsLogstashAttribute.

private LogstashAttribute preparePatternDefinitionsLogstashAttribute(final Map<String, String> patternDefinitions) {
    final LogstashAttribute logstashAttribute = mock(LogstashAttribute.class);
    final LogstashAttributeValue logstashAttributeValue = mock(LogstashAttributeValue.class);
    when(logstashAttributeValue.getAttributeValueType()).thenReturn(HASH);
    when(logstashAttributeValue.getValue()).thenReturn(patternDefinitions);
    when(logstashAttribute.getAttributeName()).thenReturn(LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME);
    when(logstashAttribute.getAttributeValue()).thenReturn(logstashAttributeValue);
    return logstashAttribute;
}