use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class DataSetExportStrategy method execute.
@Override
public StreamingResponseBody execute(ExportParameters parameters) {
final String formatName = parameters.getExportType();
final ExportFormat format = getFormat(formatName);
//
ExportUtils.setExportHeaders(//
parameters.getExportName(), //
parameters.getArguments().get(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCODING), format);
return outputStream -> {
// get the dataset content (in an auto-closable block to make sure it is properly closed)
final String datasetId = parameters.getDatasetId();
final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, datasetId, false, true);
final DataSetGetMetadata dataSetGetMetadata = applicationContext.getBean(DataSetGetMetadata.class, datasetId);
try (InputStream datasetContent = dataSetGet.execute()) {
try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
// Create dataset
final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
dataSet.setMetadata(dataSetGetMetadata.execute());
// get the actions to apply (no preparation ==> dataset export ==> no actions)
Configuration configuration = //
Configuration.builder().args(//
parameters.getArguments()).outFilter(//
rm -> filterService.build(parameters.getFilter(), rm)).format(//
format.getName()).volume(//
Configuration.Volume.SMALL).output(//
outputStream).limit(//
limit).build();
factory.get(configuration).buildExecutable(dataSet, configuration).execute();
}
} catch (TDPException e) {
throw e;
} catch (Exception e) {
throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
}
};
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class FormatRegistrationServiceTest method shouldNotGetAnyFormat.
@Test
public void shouldNotGetAnyFormat() {
final ExportFormat noFormat = service.getByName("toto");
Assert.assertNull(noFormat);
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class OptimizedExportStrategy method performOptimizedTransform.
private void performOptimizedTransform(ExportParameters parameters, OutputStream outputStream) throws IOException {
// Initial check
final OptimizedPreparationInput optimizedPreparationInput = new OptimizedPreparationInput(parameters).invoke();
if (optimizedPreparationInput == null) {
throw new IllegalStateException("Unable to use this strategy (call accept() before calling this).");
}
final String preparationId = parameters.getPreparationId();
final String dataSetId = optimizedPreparationInput.getDataSetId();
final TransformationCacheKey transformationCacheKey = optimizedPreparationInput.getTransformationCacheKey();
final DataSetMetadata metadata = optimizedPreparationInput.getMetadata();
final String previousVersion = optimizedPreparationInput.getPreviousVersion();
final String version = optimizedPreparationInput.getVersion();
final ExportFormat format = getFormat(parameters.getExportType());
// Get content from previous step
try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(contentCache.get(transformationCacheKey), UTF_8))) {
// Create dataset
final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
dataSet.setMetadata(metadata);
// get the actions to apply (no preparation ==> dataset export ==> no actions)
final String actions = getActions(preparationId, previousVersion, version);
final PreparationMessage preparation = getPreparation(preparationId);
preparation.setSteps(getMatchingSteps(preparation.getSteps(), previousVersion, version));
LOGGER.debug("Running optimized strategy for preparation {} @ step #{}", preparationId, version);
// create tee to broadcast to cache + service output
final TransformationCacheKey key = //
cacheKeyGenerator.generateContentKey(//
dataSetId, //
preparationId, //
version, //
parameters.getExportType(), //
parameters.getFrom(), //
parameters.getArguments(), //
parameters.getFilter());
LOGGER.debug("Cache key: " + key.getKey());
LOGGER.debug("Cache key details: " + key.toString());
try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
final Configuration configuration = //
Configuration.builder().args(//
parameters.getArguments()).outFilter(//
rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(//
format.getName()).actions(//
actions).preparation(//
preparation).stepId(//
version).volume(//
Configuration.Volume.SMALL).output(//
tee).limit(//
limit).build();
factory.get(configuration).buildExecutable(dataSet, configuration).execute();
tee.flush();
} catch (Throwable e) {
// NOSONAR
contentCache.evict(key);
throw e;
}
} catch (TDPException e) {
throw e;
} catch (Exception e) {
throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
}
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class PreparationExportStrategy method execute.
@Override
public StreamingResponseBody execute(final ExportParameters parameters) {
final String formatName = parameters.getExportType();
final ExportFormat format = getFormat(formatName);
//
ExportUtils.setExportHeaders(//
parameters.getExportName(), //
parameters.getArguments().get(ExportFormat.PREFIX + CSVFormat.ParametersCSV.ENCODING), format);
return outputStream -> performPreparation(parameters, outputStream);
}
use of org.talend.dataprep.format.export.ExportFormat in project data-prep by Talend.
the class FormatRegistrationServiceTest method shouldGetFormatByItsName.
@Test
public void shouldGetFormatByItsName() {
final ExportFormat csvFormat = service.getByName(CSVFormat.CSV);
Assert.assertTrue(csvFormat instanceof CSVFormat);
}
Aggregations