Examples with PreparationDTO org.talend.dataprep.api.preparation.PreparationDTO used on opensource projects

Example 1 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class OptimizedExportStrategy method performOptimizedTransform.

private void performOptimizedTransform(ExportParameters parameters, OutputStream outputStream) throws IOException {
    // Initial check
    LOGGER.debug("Prepare optimized transformation");
    final OptimizedPreparationInput optimizedPreparationInput = new OptimizedPreparationInput(parameters).invoke();
    if (optimizedPreparationInput == null) {
        throw new IllegalStateException("Unable to use this strategy (call accept() before calling this).");
    }
    LOGGER.debug("End prepare optimized transformation.");
    final String preparationId = parameters.getPreparationId();
    final String dataSetId = optimizedPreparationInput.getDataSetId();
    final TransformationCacheKey transformationCacheKey = optimizedPreparationInput.getTransformationCacheKey();
    final DataSetMetadata metadata = optimizedPreparationInput.getMetadata();
    final String previousVersion = optimizedPreparationInput.getPreviousVersion();
    final String version = optimizedPreparationInput.getVersion();
    final ExportFormat format = getFormat(parameters.getExportType());
    // Get content from previous step
    LOGGER.debug("Before get cache content");
    try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(contentCache.get(transformationCacheKey), UTF_8));
        final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser)) {
        dataSet.setMetadata(metadata);
        // get the actions to apply (no preparation ==> dataset export ==> no actions)
        final String actions = getActions(preparationId, previousVersion, version);
        final PreparationDTO preparation = getPreparation(preparationId);
        preparation.setSteps(getMatchingSteps(preparation.getSteps(), previousVersion, version));
        LOGGER.debug("Running optimized strategy for preparation {} @ step #{}", preparationId, version);
        // create tee to broadcast to cache + service output
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        dataSetId, // 
        preparationId, // 
        version, // 
        parameters.getExportType(), // 
        parameters.getFrom(), // 
        parameters.getArguments(), // 
        parameters.getFilter());
        LOGGER.debug("Cache key: " + key.getKey());
        LOGGER.debug("Cache key details: " + key.toString());
        try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
            final Configuration configuration = Configuration.builder().args(// 
            parameters.getArguments()).outFilter(// 
            rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
            format.getName()).actions(// 
            actions).preparation(// 
            preparation).stepId(// 
            version).volume(// 
            Configuration.Volume.SMALL).output(// 
            tee).limit(// 
            limit).build();
            factory.get(configuration).buildExecutable(dataSet, configuration).execute();
            tee.flush();
        } catch (Throwable e) {
            // NOSONAR
            contentCache.evict(key);
            throw e;
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    }
}

Example 2 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class TransformationServiceTest method testCache.

@Test
public void testCache() throws Exception {
    // given
    String dsId = createDataset("input_dataset.csv", "uppercase", "text/csv");
    String prepId = createEmptyPreparationFromDataset(dsId, "uppercase prep");
    applyActionFromFile(prepId, "uppercase_action.json");
    final PreparationDTO preparation = getPreparation(prepId);
    final String headId = preparation.getHeadId();
    final TransformationCacheKey key = // 
    cacheKeyGenerator.generateContentKey(// 
    dsId, // 
    preparation.getId(), // 
    headId, // 
    JSON, // 
    HEAD, // no filter
    "");
    assertFalse(contentCache.has(key));
    // when
    // 
    given().expect().statusCode(200).log().ifError().when().get("/apply/preparation/{prepId}/dataset/{datasetId}/{format}", prepId, dsId, // 
    "JSON").asString();
    // then
    assertTrue(contentCache.has(key));
    // just to pass through the cache
    final Response response = // 
    given().expect().statusCode(200).log().ifError().when().get("/apply/preparation/{prepId}/dataset/{datasetId}/{format}", prepId, dsId, "JSON");
    Assert.assertThat(response.getStatusCode(), is(200));
}

Example 3 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class OptimizedExportStrategyTest method testAcceptOK.

@Test
public void testAcceptOK() throws Exception {
    // Given
    final String datasetId = "1234";
    final String format = "";
    final String preparation = createEmptyPreparationFromDataset(datasetId, "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final PreparationDTO preparationDetails = getPreparation(preparation);
    putTransformationAndMetadataInCacheForSteps(preparationDetails.getSteps(), preparation, datasetId, format);
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setDatasetId(datasetId);
    exportParameters.setExportType(format);
    exportParameters.setFrom(HEAD);
    // Then
    assertTrue(optimizedExportStrategy.test(exportParameters));
}

Example 4 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PreparationAPI method previewAdd.

// @formatter:off
@RequestMapping(value = "/api/preparations/preview/add", method = POST, consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE)
@ApiOperation(value = "Get a preview between the head step and a new appended transformation")
public StreamingResponseBody previewAdd(@RequestBody @Valid final PreviewAddParameters input) {
    // @formatter:on
    PreparationDTO preparation = null;
    List<Action> actions = new ArrayList<>(0);
    // get preparation details with dealing with preparations
    if (StringUtils.isNotBlank(input.getPreparationId())) {
        preparation = internalGetPreparation(input.getPreparationId());
        actions = internalGetActions(preparation.getId());
    }
    final HystrixCommand<InputStream> transformation = getCommand(PreviewAdd.class, input, preparation, actions);
    return executePreviewCommand(transformation);
}

Example 5 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PreparationAPI method updatePreparationAction.

// @formatter:off
@RequestMapping(value = "/api/preparations/{preparationId}/actions/{stepId}", method = PUT, produces = APPLICATION_JSON_VALUE)
@ApiOperation(value = "Updates an action in the preparation.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.")
@Timed
public void updatePreparationAction(@ApiParam(name = "preparationId", value = "Preparation id.") @PathVariable(value = "preparationId") final String preparationId, @ApiParam(name = "stepId", value = "Step id in the preparation.") @PathVariable(value = "stepId") final String stepId, @ApiParam("New content for the action.") @RequestBody final AppendStep step) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Updating preparation action at step #{} (pool: {} )...", stepId, getConnectionStats());
    }
    // get the preparation
    PreparationDTO preparation = internalGetPreparation(preparationId);
    // get the preparation actions for up to the updated action
    final int stepIndex = new ArrayList<>(preparation.getSteps()).indexOf(stepId);
    final String parentStepId = preparation.getSteps().get(stepIndex - 1);
    final PreparationGetActions getActionsCommand = getCommand(PreparationGetActions.class, preparationId, parentStepId);
    // get the diff
    final DiffMetadata diffCommand = getCommand(DiffMetadata.class, preparation.getDataSetId(), preparationId, step.getActions(), getActionsCommand);
    // get the update action command and execute it
    final HystrixCommand<Void> command = getCommand(PreparationUpdateAction.class, preparationId, stepId, step, diffCommand);
    command.execute();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Updated preparation action at step #{} (pool: {} )...", stepId, getConnectionStats());
    }
}