Examples with ExportParameters org.talend.dataprep.api.export.ExportParameters used on opensource projects

Example 31 with ExportParameters

use of org.talend.dataprep.api.export.ExportParameters in project data-prep by Talend.

the class OptimizedExportStrategy method execute.

@Override
public StreamingResponseBody execute(ExportParameters parameters) {
    final String formatName = parameters.getExportType();
    final ExportFormat format = getFormat(formatName);
    // 
    ExportUtils.setExportHeaders(// 
    parameters.getExportName(), // 
    parameters.getArguments().get(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCODING), format);
    return outputStream -> performOptimizedTransform(parameters, outputStream);
}

Example 32 with ExportParameters

use of org.talend.dataprep.api.export.ExportParameters in project data-prep by Talend.

the class PreparationExportStrategy method performPreparation.

public void performPreparation(final ExportParameters parameters, final OutputStream outputStream) {
    final String stepId = parameters.getStepId();
    final String preparationId = parameters.getPreparationId();
    final String formatName = parameters.getExportType();
    final PreparationMessage preparation = getPreparation(preparationId, stepId);
    final String dataSetId = preparation.getDataSetId();
    final ExportFormat format = getFormat(parameters.getExportType());
    // get the dataset content (in an auto-closable block to make sure it is properly closed)
    boolean releasedIdentity = false;
    // Allow get dataset and get dataset metadata access whatever share status is
    securityProxy.asTechnicalUser();
    final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, dataSetId, false, true);
    final DataSetGetMetadata dataSetGetMetadata = applicationContext.getBean(DataSetGetMetadata.class, dataSetId);
    try (InputStream datasetContent = dataSetGet.execute()) {
        try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
            // head is not allowed as step id
            final String version = getCleanStepId(preparation, stepId);
            // Create dataset
            final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
            dataSet.setMetadata(dataSetGetMetadata.execute());
            // All good, can already release identity
            securityProxy.releaseIdentity();
            releasedIdentity = true;
            // get the actions to apply (no preparation ==> dataset export ==> no actions)
            final String actions = getActions(preparationId, version);
            final TransformationCacheKey key = // 
            cacheKeyGenerator.generateContentKey(// 
            dataSetId, // 
            preparationId, // 
            version, // 
            formatName, // 
            parameters.getFrom(), // 
            parameters.getArguments(), // 
            parameters.getFilter());
            LOGGER.debug("Cache key: " + key.getKey());
            LOGGER.debug("Cache key details: " + key.toString());
            try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
                final Configuration configuration = // 
                Configuration.builder().args(// 
                parameters.getArguments()).outFilter(// 
                rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
                format.getName()).actions(// 
                actions).preparation(// 
                preparation).stepId(// 
                version).volume(// 
                Configuration.Volume.SMALL).output(// 
                tee).limit(// 
                limit).build();
                factory.get(configuration).buildExecutable(dataSet, configuration).execute();
                tee.flush();
            } catch (Throwable e) {
                // NOSONAR
                contentCache.evict(key);
                throw e;
            }
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    } finally {
        if (!releasedIdentity) {
            // Release identity in case of error.
            securityProxy.releaseIdentity();
        }
    }
}

Example 33 with ExportParameters

use of org.talend.dataprep.api.export.ExportParameters in project data-prep by Talend.

the class SetStepRowMetadata method setStepRowMetadata.

/**
 * Update the given preparation's steps with row metadata.
 *
 * @param preparation the preparation.
 * @param currentProcessingNumber the number of the current preparation.
 * @param total the total number of preparation.
 */
private void setStepRowMetadata(PersistentPreparation preparation, AtomicLong currentProcessingNumber, long total) {
    LOGGER.info("[{}/{}] preparation #{} migration starting...", currentProcessingNumber.addAndGet(1), total, preparation.getId());
    // Check if preparation is accessible to end user
    final Folder folder = folderRepository.locateEntry(preparation.id(), PREPARATION);
    if (folder == null) {
        LOGGER.warn("Preparation {} does not belong to a folder, skip migration (not accessible to user).", preparation.getName());
        return;
    }
    // Run preparation
    final ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation.getId());
    exportParameters.setDatasetId(preparation.getDataSetId());
    exportParameters.setExportType("JSON");
    // just process the preparation, the transformation service will automatically update the steps with row metadata
    try (NullOutputStream outputStream = new NullOutputStream()) {
        service.execute(exportParameters).writeTo(outputStream);
    } catch (Throwable e) {
        LOGGER.warn("Error processing preparation {} (#{}), semantic categories are not properly stored and may change if you change them using the data quality command line", preparation.getName(), preparation.getId());
        LOGGER.debug("Here is the stacktrace", e);
    }
    LOGGER.info("[{}/{}] preparation #{} done", currentProcessingNumber.get(), total, preparation.getId());
}

Example 34 with ExportParameters

use of org.talend.dataprep.api.export.ExportParameters in project data-prep by Talend.

the class PreparationExportStrategyTest method shouldUsedHeadPreparation.

@Test
public void shouldUsedHeadPreparation() throws IOException {
    // Given
    final ExportParameters parameters = new ExportParameters();
    parameters.setExportType("JSON");
    parameters.setPreparationId("prep-1234");
    parameters.setStepId("head");
    final Preparation preparation = new Preparation();
    preparation.setId("prep-1234");
    preparation.setHeadId("head");
    configurePreparation(preparation, "prep-1234", "head");
    // When
    final StreamingResponseBody body = strategy.execute(parameters);
    body.writeTo(new NullOutputStream());
    // Then
    final ArgumentCaptor<Configuration> captor = ArgumentCaptor.forClass(Configuration.class);
    verify(transformer).buildExecutable(any(), captor.capture());
    assertEquals("prep-1234", captor.getValue().getPreparationId());
    assertEquals("head", captor.getValue().getPreparation().getHeadId());
}

Example 35 with ExportParameters

use of org.talend.dataprep.api.export.ExportParameters in project data-prep by Talend.

the class PreparationExportStrategyTest method shouldAcceptParameters.

@Test
public void shouldAcceptParameters() {
    // Then
    final ExportParameters parameters = new ExportParameters();
    parameters.setContent(null);
    parameters.setFrom(null);
    parameters.setFrom(ExportParameters.SourceType.HEAD);
    parameters.setPreparationId("prep-1234");
    parameters.setDatasetId("");
    assertTrue(strategy.accept(parameters));
}