Examples with PreparationDTO org.talend.dataprep.api.preparation.PreparationDTO used on opensource projects

Example 26 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class StandardExportStrategiesIntegrationTest method idOfPrepWith2StepsOrMore.

private String idOfPrepWith2StepsOrMore() throws IOException {
    reset(preparationDetailsGet);
    final PreparationDetailsDTO preparationDetailsDTO = mapper.readerFor(// 
    PreparationDetailsDTO.class).readValue(this.getClass().getResourceAsStream("two_steps_preparation_details.json"));
    // 
    when(preparationDetailsGet.execute()).thenReturn(// 
    preparationDetailsDTO).thenReturn(preparationDetailsDTO);
    final PreparationDTO preparationDTO = mapper.readerFor(// 
    PreparationDTO.class).readValue(this.getClass().getResourceAsStream("two_steps_preparation_details_summary.json"));
    // 
    when(preparationSummaryGet.execute()).thenReturn(// 
    preparationDTO).thenReturn(preparationDTO);
    return "prepId-1234";
}

Example 27 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PreparationExportStrategy method performPreparation.

public void performPreparation(final ExportParameters parameters, final OutputStream outputStream) {
    final String stepId = parameters.getStepId();
    final String preparationId = parameters.getPreparationId();
    final String formatName = parameters.getExportType();
    final PreparationDTO preparation = getPreparation(preparationId, stepId);
    final String dataSetId = preparation.getDataSetId();
    final ExportFormat format = getFormat(parameters.getExportType());
    boolean releasedIdentity = false;
    // Allow get dataset and get dataset metadata access whatever share status is
    securityProxy.asTechnicalUserForDataSet();
    try (DataSet dataSet = datasetClient.getDataSet(dataSetId, false, true)) {
        // head is not allowed as step id
        final String version = getCleanStepId(preparation, stepId);
        // All good, can already release identity
        securityProxy.releaseIdentity();
        releasedIdentity = true;
        // get the actions to apply (no preparation ==> dataset export ==> no actions)
        final String actions = getActions(preparationId, version);
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        dataSetId, // 
        preparationId, // 
        version, // 
        formatName, // 
        parameters.getFrom(), // 
        parameters.getArguments(), // 
        parameters.getFilter());
        LOGGER.debug("Cache key: {}", key.getKey());
        LOGGER.debug("Cache key details: {}", key.toString());
        try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
            final Configuration configuration = Configuration.builder().args(// 
            parameters.getArguments()).outFilter(// 
            rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
            format.getName()).actions(// 
            actions).preparation(// 
            preparation).stepId(// 
            version).volume(// 
            Configuration.Volume.SMALL).output(// 
            tee).limit(// 
            limit).build();
            factory.get(configuration).buildExecutable(dataSet, configuration).execute();
            tee.flush();
        } catch (Throwable e) {
            // NOSONAR
            contentCache.evict(key);
            throw e;
        }
    } catch (TalendRuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    } finally {
        if (!releasedIdentity) {
            // Release identity in case of error.
            securityProxy.releaseIdentity();
        }
    }
}

Example 28 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class OptimizedExportStrategyTest method testAcceptKO_stepIdIsRootStep.

@Test
public void testAcceptKO_stepIdIsRootStep() throws Exception {
    // Given
    final String datasetId = "1234";
    final String format = "";
    final String preparation = createEmptyPreparationFromDataset(datasetId, "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final PreparationDTO preparationDetails = getPreparation(preparation);
    final List<String> stepsWithoutRootStep = new ArrayList<>(preparationDetails.getSteps());
    stepsWithoutRootStep.remove(0);
    putTransformationAndMetadataInCacheForSteps(stepsWithoutRootStep, preparation, datasetId, format);
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setDatasetId(datasetId);
    exportParameters.setExportType(format);
    exportParameters.setFrom(HEAD);
    exportParameters.setStepId(Step.ROOT_STEP.id());
    // Then
    assertFalse("The OptimizedExportStrategy should not be acceptable if version is ROOT_STEP", optimizedExportStrategy.test(exportParameters));
}

Example 29 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class OptimizedExportStrategyTest method testExecute.

@Test
public void testExecute() throws Exception {
    // Given
    final String datasetId = "1234";
    final String format = "JSON";
    final String preparation = createEmptyPreparationFromDataset(datasetId, "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final PreparationDTO preparationDetails = getPreparation(preparation);
    for (String step : preparationDetails.getSteps()) {
        try (OutputStream content = contentCache.put(cacheKeyGenerator.generateMetadataKey(preparation, step, HEAD), ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        datasetId, // 
        preparation, // 
        step, // 
        format, // 
        HEAD, // no filter
        "");
        try (OutputStream content = contentCache.put(key, ContentCache.TimeToLive.DEFAULT)) {
            content.write("{\"records\": [{\"0000\": \"a\"}]}".getBytes());
            content.flush();
        }
    }
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setDatasetId(datasetId);
    exportParameters.setExportType(format);
    exportParameters.setFrom(HEAD);
    // Then
    optimizedExportStrategy.execute(exportParameters);
}

Example 30 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class OptimizedExportStrategyTest method testAcceptKO_withMetadataCacheNoContentCache.

@Test
public void testAcceptKO_withMetadataCacheNoContentCache() throws Exception {
    // Given
    final String preparation = createEmptyPreparationFromDataset("1234", "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final PreparationDTO preparationDetails = getPreparation(preparation);
    for (String step : preparationDetails.getSteps()) {
        try (OutputStream content = contentCache.put(cacheKeyGenerator.generateMetadataKey(preparation, step, HEAD), ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
    }
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setFrom(HEAD);
    // Then
    assertFalse(optimizedExportStrategy.test(exportParameters));
}