Examples with PipelinesDataFlowModel com.amazon.dataprepper.model.configuration.PipelinesDataFlowModel used on opensource projects

Example 1 with PipelinesDataFlowModel

use of com.amazon.dataprepper.model.configuration.PipelinesDataFlowModel in project data-prepper by opensearch-project.

the class LogstashConfigConverter method convertLogstashConfigurationToPipeline.

public String convertLogstashConfigurationToPipeline(String logstashConfigurationPath, String outputDirectory) throws IOException {
    final Path configurationFilePath = Paths.get(logstashConfigurationPath);
    final String logstashConfigAsString = new String(Files.readAllBytes(configurationFilePath));
    LogstashLexer lexer = new LogstashLexer(CharStreams.fromString(logstashConfigAsString));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    LogstashParser parser = new LogstashParser(tokens);
    final ParseTree tree = parser.config();
    ModelConvertingLogstashVisitor visitor = new ModelConvertingLogstashVisitor();
    LogstashConfiguration logstashConfiguration = (LogstashConfiguration) visitor.visit(tree);
    LogstashMapper logstashMapper = new LogstashMapper();
    PipelineModel pipelineModel = logstashMapper.mapPipeline(logstashConfiguration);
    PipelinesDataFlowModel pipelinesDataFlowModel = new PipelinesDataFlowModel(Collections.singletonMap("logstash-converted-pipeline", pipelineModel));
    ObjectMapper mapper = new ObjectMapper(YAMLFactory.builder().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER).enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR).enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS).disable(YAMLGenerator.Feature.SPLIT_LINES).enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE).build());
    final String confFileName = configurationFilePath.getFileName().toString();
    final String yamlFileName = confFileName.substring(0, confFileName.lastIndexOf(".conf"));
    final Path yamlFilePath = Paths.get(outputDirectory, yamlFileName + ".yaml");
    mapper.writeValue(new File(String.valueOf(yamlFilePath)), pipelinesDataFlowModel);
    return String.valueOf(yamlFilePath);
}

Example 2 with PipelinesDataFlowModel

use of com.amazon.dataprepper.model.configuration.PipelinesDataFlowModel in project data-prepper by opensearch-project.

the class LogstashConfigConverterIT method convertLogstashConfigurationToPipeline_should_return_valid_PipelinesDataFlowModel_with_the_single_known_pipeline.

@ParameterizedTest
@ArgumentsSource(LogstashPathsProviders.class)
void convertLogstashConfigurationToPipeline_should_return_valid_PipelinesDataFlowModel_with_the_single_known_pipeline(final String configurationPath) throws IOException {
    final String actualPath = createObjectUnderTest().convertLogstashConfigurationToPipeline(configurationPath, OUTPUT_DIRECTORY);
    assertThat(actualPath, notNullValue());
    final String dataPrepperConfigurationString = Files.readString(Path.of(actualPath));
    assertThat(dataPrepperConfigurationString, notNullValue());
    final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
    final PipelinesDataFlowModel pipelinesDataFlowModel = objectMapper.readValue(dataPrepperConfigurationString, PipelinesDataFlowModel.class);
    assertThat(pipelinesDataFlowModel, notNullValue());
    assertThat(pipelinesDataFlowModel.getPipelines(), notNullValue());
    assertThat(pipelinesDataFlowModel.getPipelines().size(), equalTo(1));
    assertThat(pipelinesDataFlowModel.getPipelines(), hasKey("logstash-converted-pipeline"));
    assertThat(pipelinesDataFlowModel.getPipelines().get("logstash-converted-pipeline"), notNullValue());
}