use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class OptimizedExportStrategy method execute.
@Override
public StreamingResponseBody execute(ExportParameters parameters) {
final String formatName = parameters.getExportType();
final ExportFormat format = getFormat(formatName);
//
ExportUtils.setExportHeaders(//
parameters.getExportName(), //
parameters.getArguments().get(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCODING), format);
return outputStream -> performOptimizedTransform(parameters, outputStream);
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class PreparationExportStrategy method performPreparation.
public void performPreparation(final ExportParameters parameters, final OutputStream outputStream) {
final String stepId = parameters.getStepId();
final String preparationId = parameters.getPreparationId();
final String formatName = parameters.getExportType();
final PreparationMessage preparation = getPreparation(preparationId, stepId);
final String dataSetId = preparation.getDataSetId();
final ExportFormat format = getFormat(parameters.getExportType());
// get the dataset content (in an auto-closable block to make sure it is properly closed)
boolean releasedIdentity = false;
// Allow get dataset and get dataset metadata access whatever share status is
securityProxy.asTechnicalUser();
final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, dataSetId, false, true);
final DataSetGetMetadata dataSetGetMetadata = applicationContext.getBean(DataSetGetMetadata.class, dataSetId);
try (InputStream datasetContent = dataSetGet.execute()) {
try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
// head is not allowed as step id
final String version = getCleanStepId(preparation, stepId);
// Create dataset
final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
dataSet.setMetadata(dataSetGetMetadata.execute());
// All good, can already release identity
securityProxy.releaseIdentity();
releasedIdentity = true;
// get the actions to apply (no preparation ==> dataset export ==> no actions)
final String actions = getActions(preparationId, version);
final TransformationCacheKey key = //
cacheKeyGenerator.generateContentKey(//
dataSetId, //
preparationId, //
version, //
formatName, //
parameters.getFrom(), //
parameters.getArguments(), //
parameters.getFilter());
LOGGER.debug("Cache key: " + key.getKey());
LOGGER.debug("Cache key details: " + key.toString());
try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
final Configuration configuration = //
Configuration.builder().args(//
parameters.getArguments()).outFilter(//
rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(//
format.getName()).actions(//
actions).preparation(//
preparation).stepId(//
version).volume(//
Configuration.Volume.SMALL).output(//
tee).limit(//
limit).build();
factory.get(configuration).buildExecutable(dataSet, configuration).execute();
tee.flush();
} catch (Throwable e) {
// NOSONAR
contentCache.evict(key);
throw e;
}
}
} catch (TDPException e) {
throw e;
} catch (Exception e) {
throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
} finally {
if (!releasedIdentity) {
// Release identity in case of error.
securityProxy.releaseIdentity();
}
}
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class ApplyPreparationExportStrategy method execute.
@Override
public StreamingResponseBody execute(ExportParameters parameters) {
final String formatName = parameters.getExportType();
final ExportFormat format = getFormat(formatName);
//
ExportUtils.setExportHeaders(//
parameters.getExportName(), //
parameters.getArguments().get(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCODING), format);
return outputStream -> executeApplyPreparation(parameters, outputStream);
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class ApplyPreparationExportStrategy method executeApplyPreparation.
private void executeApplyPreparation(ExportParameters parameters, OutputStream outputStream) {
final String stepId = parameters.getStepId();
final String preparationId = parameters.getPreparationId();
final String formatName = parameters.getExportType();
final Preparation preparation = getPreparation(preparationId);
final String dataSetId = parameters.getDatasetId();
final ExportFormat format = getFormat(parameters.getExportType());
// dataset content must be retrieved as the technical user because it might not be shared
boolean technicianIdentityReleased = false;
securityProxy.asTechnicalUser();
// get the dataset content (in an auto-closable block to make sure it is properly closed)
final boolean fullContent = parameters.getFrom() == ExportParameters.SourceType.FILTER;
final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, dataSetId, fullContent, true);
try (final InputStream datasetContent = dataSetGet.execute();
final JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
// release the technical user identity
securityProxy.releaseIdentity();
technicianIdentityReleased = true;
// head is not allowed as step id
final String version = getCleanStepId(preparation, stepId);
// Create dataset
final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
// get the actions to apply (no preparation ==> dataset export ==> no actions)
final String actions = getActions(preparationId, version);
// create tee to broadcast to cache + service output
final TransformationCacheKey key = //
cacheKeyGenerator.generateContentKey(//
dataSetId, //
preparationId, //
version, //
formatName, //
parameters.getFrom(), //
parameters.getArguments(), //
parameters.getFilter());
LOGGER.debug("Cache key: " + key.getKey());
LOGGER.debug("Cache key details: " + key.toString());
try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
final Configuration.Builder configurationBuilder = //
Configuration.builder().args(//
parameters.getArguments()).outFilter(//
rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(//
format.getName()).actions(//
actions).preparation(//
getPreparation(preparationId)).stepId(//
version).volume(//
SMALL).output(//
tee).limit(this.limit);
// no need for statistics if it's not JSON output
if (!Objects.equals(format.getName(), JSON)) {
configurationBuilder.globalStatistics(false);
}
final Configuration configuration = configurationBuilder.build();
factory.get(configuration).buildExecutable(dataSet, configuration).execute();
tee.flush();
} catch (Throwable e) {
// NOSONAR
LOGGER.debug("evicting cache {}", key.getKey());
contentCache.evict(key);
throw e;
}
} catch (TDPException e) {
throw e;
} catch (Exception e) {
throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
} finally {
if (!technicianIdentityReleased) {
securityProxy.releaseIdentity();
}
}
}
Aggregations