Examples with PluginModel com.amazon.dataprepper.model.configuration.PluginModel used on opensource projects

Example 1 with PluginModel

use of com.amazon.dataprepper.model.configuration.PluginModel in project data-prepper by opensearch-project.

the class MutateMapper method mapAttributes.

public List<PluginModel> mapAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings) {
    List<PluginModel> models = new LinkedList<>();
    List<AddEntryConfig> adds = new LinkedList<>();
    List<RenameCopyConfig> renames = new LinkedList<>();
    List<String> deletes = new LinkedList<>();
    List<RenameCopyConfig> copies = new LinkedList<>();
    List<String> uppercases = new LinkedList<>();
    for (LogstashAttribute attr : logstashAttributes) {
        final String name = attr.getAttributeName();
        if (Objects.equals(name, "add_field")) {
            ((Map<String, Object>) attr.getAttributeValue().getValue()).forEach((key, value) -> adds.add(new AddEntryConfig(NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(key), value)));
        } else if (Objects.equals(name, "rename")) {
            ((Map<String, String>) attr.getAttributeValue().getValue()).forEach((key, value) -> renames.add(new RenameCopyConfig(NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(key), NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(value))));
        } else if (Objects.equals(name, "remove_field")) {
            deletes.addAll(((List<String>) attr.getAttributeValue().getValue()).stream().map(NestedSyntaxConverter::convertNestedSyntaxToJsonPointer).collect(Collectors.toList()));
        } else if (Objects.equals(name, "copy")) {
            ((Map<String, String>) attr.getAttributeValue().getValue()).forEach((key, value) -> copies.add(new RenameCopyConfig(NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(key), NestedSyntaxConverter.convertNestedSyntaxToJsonPointer(value))));
        } else if (Objects.equals(name, "uppercase")) {
            uppercases.addAll(((ArrayList<String>) attr.getAttributeValue().getValue()).stream().map(NestedSyntaxConverter::convertNestedSyntaxToJsonPointer).collect(Collectors.toList()));
        }
    }
    if (!renames.isEmpty()) {
        Map<String, Object> renameMap = new HashMap<>();
        renameMap.put("entries", renames);
        PluginModel renameModel = new PluginModel("rename_keys", renameMap);
        models.add(renameModel);
    }
    if (!copies.isEmpty()) {
        Map<String, Object> copyMap = new HashMap<>();
        copyMap.put("entries", copies);
        PluginModel renameModel = new PluginModel("copy_values", copyMap);
        models.add(renameModel);
    }
    if (!adds.isEmpty()) {
        Map<String, Object> addMap = new HashMap<>();
        addMap.put("entries", adds);
        PluginModel addModel = new PluginModel("add_entries", addMap);
        models.add(addModel);
    }
    if (!deletes.isEmpty()) {
        Map<String, Object> deleteMap = new HashMap<>();
        deleteMap.put("with_keys", deletes);
        PluginModel deleteModel = new PluginModel("delete_entries", deleteMap);
        models.add(deleteModel);
    }
    if (!uppercases.isEmpty()) {
        Map<String, Object> uppercaseMap = new HashMap<>();
        uppercaseMap.put("with_keys", uppercases);
        PluginModel uppercaseModel = new PluginModel("uppercase_string", uppercaseMap);
        models.add(uppercaseModel);
    }
    return models;
}

Example 2 with PluginModel

use of com.amazon.dataprepper.model.configuration.PluginModel in project data-prepper by opensearch-project.

the class LogstashPluginMapper method mapPlugin.

public List<PluginModel> mapPlugin(LogstashPlugin logstashPlugin) {
    final String mappingResourceName = logstashPlugin.getPluginName() + ".mapping.yaml";
    final InputStream inputStream = this.getClass().getResourceAsStream(mappingResourceName);
    if (inputStream == null) {
        throw new LogstashMappingException("Unable to find mapping resource " + mappingResourceName);
    }
    LogstashMappingModel logstashMappingModel;
    try {
        logstashMappingModel = objectMapper.readValue(inputStream, LogstashMappingModel.class);
    } catch (IOException ex) {
        throw new LogstashMappingException("Unable to parse mapping file " + mappingResourceName, ex);
    }
    if (logstashMappingModel.getCustomPluginMapperClass() != null) {
        final LogstashPluginAttributesMapper mapper = pluginMapperProvider.getAttributesMapper(logstashMappingModel);
        return mapper.mapAttributes(logstashPlugin.getAttributes(), logstashMappingModel);
    }
    if (logstashMappingModel.getPluginName() == null) {
        throw new LogstashMappingException("The mapping file " + mappingResourceName + " has a null value for 'pluginName'.");
    }
    final LogstashPluginAttributesMapper pluginAttributesMapper = pluginMapperProvider.getAttributesMapper(logstashMappingModel);
    final List<PluginModel> pluginModels = pluginAttributesMapper.mapAttributes(logstashPlugin.getAttributes(), logstashMappingModel);
    return pluginModels;
}

Example 3 with PluginModel

use of com.amazon.dataprepper.model.configuration.PluginModel in project data-prepper by opensearch-project.

the class LogstashMapper method mapPipeline.

public PipelineModel mapPipeline(LogstashConfiguration logstashConfiguration) {
    List<PluginModel> sourcePluginModels = mapPluginSection(logstashConfiguration, LogstashPluginType.INPUT);
    PluginModel sourcePlugin = null;
    if (sourcePluginModels.size() > 1)
        throw new LogstashMappingException("More than 1 source plugins are not supported");
    else if (sourcePluginModels.size() == 1)
        sourcePlugin = sourcePluginModels.get(0);
    List<PluginModel> prepperPluginModels = mapPluginSection(logstashConfiguration, LogstashPluginType.FILTER);
    List<PluginModel> sinkPluginModels = mapPluginSection(logstashConfiguration, LogstashPluginType.OUTPUT);
    return new PipelineModel(sourcePlugin, prepperPluginModels, sinkPluginModels, null, null);
}

Example 4 with PluginModel

use of com.amazon.dataprepper.model.configuration.PluginModel in project data-prepper by opensearch-project.

the class GrokLogstashPluginAttributesMapperTest method mapAttributes_converts_nested_syntax_in_mapped_attributes.

@Test
void mapAttributes_converts_nested_syntax_in_mapped_attributes() {
    final LogstashAttribute matchMultiKeysLogstashAttribute = prepareHashTypeMatchLogstashAttribute(Arrays.asList(Map.entry("[text][message]", "fake message regex 1"), Map.entry("other", "fake other regex")));
    final LogstashAttribute matchMessageLogstashAttribute2 = prepareArrayTypeMatchLogstashAttribute("[text][message]", "%{NUMBER} %{GREEDYDATA:[nested][field][data2]}");
    final List<LogstashAttribute> matchLogstashAttributes = Arrays.asList(matchMultiKeysLogstashAttribute, matchMessageLogstashAttribute2);
    final Map<String, Object> expectedMatchSettings = Map.of("/text/message", Arrays.asList("fake message regex 1", "%{NUMBER} %{GREEDYDATA:/nested/field/data2}"), "other", Collections.singletonList("fake other regex"));
    final String dataPrepperMatchAttribute = "match";
    final LogstashAttributesMappings mappings = mock(LogstashAttributesMappings.class);
    when(mappings.getMappedAttributeNames()).thenReturn(Collections.singletonMap(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, dataPrepperMatchAttribute));
    final List<PluginModel> actualPluginModels = createObjectUnderTest().mapAttributes(matchLogstashAttributes, mappings);
    assertThat(actualPluginModels, notNullValue());
    assertThat(actualPluginModels.size(), equalTo(1));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperMatchAttribute));
    assertThat(actualPluginModels.get(0).getPluginSettings().get(dataPrepperMatchAttribute), equalTo(expectedMatchSettings));
}

Example 5 with PluginModel

use of com.amazon.dataprepper.model.configuration.PluginModel in project data-prepper by opensearch-project.

the class GrokLogstashPluginAttributesMapperTest method mapAttributes_sets_mapped_attributes_match_with_named_captures_and_pattern_definitions.

@SuppressWarnings("unchecked")
@Test
void mapAttributes_sets_mapped_attributes_match_with_named_captures_and_pattern_definitions() {
    final String testNamedCapture = "test_named_capture";
    final String testRegex = "fake_regex";
    final String handWrittenPatternName = "handwritten_pattern_name";
    final String handWrittenPatternValue = "handwritten_pattern_value";
    final LogstashAttribute matchMessageLogstashAttributeWithNamedCaptures = prepareArrayTypeMatchLogstashAttribute("message", String.format("(?<%s>%s)", testNamedCapture, testRegex));
    final LogstashAttribute patternDefinitionsLogstashAttribute = preparePatternDefinitionsLogstashAttribute(Map.of(handWrittenPatternName, handWrittenPatternValue));
    final String dataPrepperMatchAttribute = "match";
    final String dataPrepperPatternDefinitionsAttribute = "pattern_definitions";
    final LogstashAttributesMappings mappings = mock(LogstashAttributesMappings.class);
    when(mappings.getMappedAttributeNames()).thenReturn(Map.of(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, dataPrepperMatchAttribute, LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME, dataPrepperPatternDefinitionsAttribute));
    final List<PluginModel> actualPluginModels = createObjectUnderTest().mapAttributes(Arrays.asList(matchMessageLogstashAttributeWithNamedCaptures, patternDefinitionsLogstashAttribute), mappings);
    assertThat(actualPluginModels, notNullValue());
    assertThat(actualPluginModels.size(), equalTo(1));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperMatchAttribute));
    assertThat(actualPluginModels.get(0).getPluginSettings(), hasKey(dataPrepperPatternDefinitionsAttribute));
    final Map<String, String> actualPatternDefinitions = (Map<String, String>) actualPluginModels.get(0).getPluginSettings().get(dataPrepperPatternDefinitionsAttribute);
    assertThat(actualPatternDefinitions, hasKey(handWrittenPatternName));
    assertThat(actualPatternDefinitions, hasValue(handWrittenPatternValue));
    assertThat(actualPatternDefinitions, hasValue(testRegex));
    final Map<String, List<String>> actualMatch = (Map<String, List<String>>) actualPluginModels.get(0).getPluginSettings().get(dataPrepperMatchAttribute);
    assertThat(actualMatch.get("message").get(0), matchesPattern(String.format("%%\\{(.*?):%s\\}", testNamedCapture)));
}