Examples with Configuration org.talend.dataprep.transformation.api.transformer.configuration.Configuration used on opensource projects

Example 1 with Configuration

use of org.talend.dataprep.transformation.api.transformer.configuration.Configuration in project data-prep by Talend.

the class DataSetExportStrategy method execute.

@Override
public StreamingResponseBody execute(ExportParameters parameters) {
    final String formatName = parameters.getExportType();
    final ExportFormat format = getFormat(formatName);
    // 
    ExportUtils.setExportHeaders(// 
    parameters.getExportName(), // 
    parameters.getArguments().get(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCODING), format);
    return outputStream -> {
        // get the dataset content (in an auto-closable block to make sure it is properly closed)
        final String datasetId = parameters.getDatasetId();
        final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, datasetId, false, true);
        final DataSetGetMetadata dataSetGetMetadata = applicationContext.getBean(DataSetGetMetadata.class, datasetId);
        try (InputStream datasetContent = dataSetGet.execute()) {
            try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
                // Create dataset
                final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
                dataSet.setMetadata(dataSetGetMetadata.execute());
                // get the actions to apply (no preparation ==> dataset export ==> no actions)
                Configuration configuration = // 
                Configuration.builder().args(// 
                parameters.getArguments()).outFilter(// 
                rm -> filterService.build(parameters.getFilter(), rm)).format(// 
                format.getName()).volume(// 
                Configuration.Volume.SMALL).output(// 
                outputStream).limit(// 
                limit).build();
                factory.get(configuration).buildExecutable(dataSet, configuration).execute();
            }
        } catch (TDPException e) {
            throw e;
        } catch (Exception e) {
            throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
        }
    };
}

Example 2 with Configuration

use of org.talend.dataprep.transformation.api.transformer.configuration.Configuration in project data-prep by Talend.

the class OptimizedExportStrategy method performOptimizedTransform.

private void performOptimizedTransform(ExportParameters parameters, OutputStream outputStream) throws IOException {
    // Initial check
    final OptimizedPreparationInput optimizedPreparationInput = new OptimizedPreparationInput(parameters).invoke();
    if (optimizedPreparationInput == null) {
        throw new IllegalStateException("Unable to use this strategy (call accept() before calling this).");
    }
    final String preparationId = parameters.getPreparationId();
    final String dataSetId = optimizedPreparationInput.getDataSetId();
    final TransformationCacheKey transformationCacheKey = optimizedPreparationInput.getTransformationCacheKey();
    final DataSetMetadata metadata = optimizedPreparationInput.getMetadata();
    final String previousVersion = optimizedPreparationInput.getPreviousVersion();
    final String version = optimizedPreparationInput.getVersion();
    final ExportFormat format = getFormat(parameters.getExportType());
    // Get content from previous step
    try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(contentCache.get(transformationCacheKey), UTF_8))) {
        // Create dataset
        final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
        dataSet.setMetadata(metadata);
        // get the actions to apply (no preparation ==> dataset export ==> no actions)
        final String actions = getActions(preparationId, previousVersion, version);
        final PreparationMessage preparation = getPreparation(preparationId);
        preparation.setSteps(getMatchingSteps(preparation.getSteps(), previousVersion, version));
        LOGGER.debug("Running optimized strategy for preparation {} @ step #{}", preparationId, version);
        // create tee to broadcast to cache + service output
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        dataSetId, // 
        preparationId, // 
        version, // 
        parameters.getExportType(), // 
        parameters.getFrom(), // 
        parameters.getArguments(), // 
        parameters.getFilter());
        LOGGER.debug("Cache key: " + key.getKey());
        LOGGER.debug("Cache key details: " + key.toString());
        try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
            final Configuration configuration = // 
            Configuration.builder().args(// 
            parameters.getArguments()).outFilter(// 
            rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
            format.getName()).actions(// 
            actions).preparation(// 
            preparation).stepId(// 
            version).volume(// 
            Configuration.Volume.SMALL).output(// 
            tee).limit(// 
            limit).build();
            factory.get(configuration).buildExecutable(dataSet, configuration).execute();
            tee.flush();
        } catch (Throwable e) {
            // NOSONAR
            contentCache.evict(key);
            throw e;
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    }
}

Example 3 with Configuration

use of org.talend.dataprep.transformation.api.transformer.configuration.Configuration in project data-prep by Talend.

the class TransformerFactoryTest method getExporter_csv_exporter_should_write_csv_format.

@Test
public void getExporter_csv_exporter_should_write_csv_format() throws Exception {
    // given
    Map<String, String> arguments = new HashMap<>();
    arguments.put(ExportFormat.PREFIX + CSVFormat.ParametersCSV.FIELDS_DELIMITER, ";");
    arguments.put(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCLOSURE_MODE, CSVFormat.ParametersCSV.ENCLOSURE_ALL_FIELDS);
    final OutputStream outputStream = new ByteArrayOutputStream();
    final Configuration configuration = // 
    Configuration.builder().args(// 
    arguments).format(// 
    CSV).output(// 
    outputStream).actions(// 
    IOUtils.toString(TransformerFactoryTest.class.getResourceAsStream("upper_case_firstname.json"), UTF_8)).build();
    final Transformer transformer = factory.get(configuration);
    final String expectedCsv = IOUtils.toString(TransformerFactoryTest.class.getResourceAsStream("expected_export_preparation_uppercase_firstname.csv"), UTF_8);
    final InputStream inputStream = TransformerFactoryTest.class.getResourceAsStream("../../format/export_dataset.json");
    try (JsonParser parser = mapper.getFactory().createParser(inputStream)) {
        final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
        // when
        transformer.buildExecutable(dataSet, configuration).execute();
        // then
        assertThat(outputStream.toString()).isEqualTo(expectedCsv);
    }
}

Example 4 with Configuration

use of org.talend.dataprep.transformation.api.transformer.configuration.Configuration in project data-prep by Talend.

the class PreparationExportStrategyTest method shouldUsedVersionedPreparation.

@Test
public void shouldUsedVersionedPreparation() throws IOException {
    // Given
    final ExportParameters parameters = new ExportParameters();
    parameters.setExportType("JSON");
    parameters.setPreparationId("prep-1234");
    parameters.setStepId("step-1234");
    final Preparation preparation = new Preparation();
    preparation.setId("prep-1234");
    preparation.setHeadId("step-1234");
    configurePreparation(preparation, "prep-1234", "step-1234");
    // When
    final StreamingResponseBody body = strategy.execute(parameters);
    body.writeTo(new NullOutputStream());
    // Then
    final ArgumentCaptor<Configuration> captor = ArgumentCaptor.forClass(Configuration.class);
    verify(transformer).buildExecutable(any(), captor.capture());
    assertEquals("prep-1234", captor.getValue().getPreparationId());
    assertEquals("step-1234", captor.getValue().getPreparation().getHeadId());
}

Example 5 with Configuration

use of org.talend.dataprep.transformation.api.transformer.configuration.Configuration in project data-prep by Talend.

the class XlsWriterTest method createSchemaParser.

/**
 * utility function
 */
public SchemaParser.Request createSchemaParser(String inputFileName) throws Exception {
    Path path = Files.createTempFile("datarep-foo", "xlsx");
    Files.deleteIfExists(path);
    try (final OutputStream outputStream = Files.newOutputStream(path)) {
        final Configuration configuration = // 
        Configuration.builder().format(// 
        XlsFormat.XLSX).output(// 
        outputStream).actions(// 
        "").build();
        final Transformer exporter = factory.get(configuration);
        final InputStream inputStream = XlsWriterTest.class.getResourceAsStream(inputFileName);
        try (JsonParser parser = mapper.getFactory().createParser(inputStream)) {
            final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
            exporter.buildExecutable(dataSet, configuration).execute();
        }
    }
    DataSetMetadata metadata = metadataBuilder.metadata().id("123").build();
    return new SchemaParser.Request(Files.newInputStream(path), metadata);
}