Search in sources :

Example 1 with PreparationMessage

use of org.talend.dataprep.api.preparation.PreparationMessage in project data-prep by Talend.

the class APIPreparationConversionsTest method shouldDealWithExceptionInDataSetGet.

@Test
public void shouldDealWithExceptionInDataSetGet() {
    // given
    final PreparationMessage preparation = getPreparationMessage("DS-1234");
    when(applicationContext.getBean(eq(DataSetGet.class), any(Object[].class))).thenAnswer(i -> {
        final DataSetGet mock = mock(DataSetGet.class);
        when(mock.execute()).thenThrow(new TDPException(DataSetErrorCodes.DATASET_DOES_NOT_EXIST));
        return mock;
    });
    // when
    final EnrichedPreparation actual = conversionService.convert(preparation, EnrichedPreparation.class);
    // then
    assertNotNull(actual);
    assertNull(actual.getSummary());
    assertNull(actual.getFolder());
}
Also used : TDPException(org.talend.dataprep.exception.TDPException) DataSetGet(org.talend.dataprep.command.dataset.DataSetGet) EnrichedPreparation(org.talend.dataprep.api.service.api.EnrichedPreparation) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) Test(org.junit.Test)

Example 2 with PreparationMessage

use of org.talend.dataprep.api.preparation.PreparationMessage in project data-prep by Talend.

the class APIPreparationConversionsTest method shouldEnrichPreparationWithDataset.

@Test
public void shouldEnrichPreparationWithDataset() {
    // given
    DataSetMetadata metadata = getDataSetMetadata("super dataset", 1001L);
    setupHystrixCommand(DataSetGetMetadata.class, metadata);
    final PreparationMessage preparation = getPreparationMessage(metadata.getId());
    Folder folder = getFolder("F-753854");
    setupHystrixCommand(LocatePreparation.class, folder);
    // when
    final EnrichedPreparation actual = conversionService.convert(preparation, EnrichedPreparation.class);
    // then
    assertEquals(metadata.getId(), actual.getSummary().getDataSetId());
    assertEquals(metadata.getName(), actual.getSummary().getDataSetName());
    assertEquals(metadata.getContent().getNbRecords(), actual.getSummary().getDataSetNbRow());
    final List<String> expectedSteps = preparation.getSteps().stream().map(Step::getId).collect(Collectors.toList());
    final List<String> actualSteps = actual.getSteps();
    assertNotNull(actualSteps);
    assertEquals(expectedSteps.size(), expectedSteps.size());
    expectedSteps.forEach(s -> assertTrue(actualSteps.contains(s)));
    assertEquals(folder, actual.getFolder());
}
Also used : EnrichedPreparation(org.talend.dataprep.api.service.api.EnrichedPreparation) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) Folder(org.talend.dataprep.api.folder.Folder) DataSetMetadata(org.talend.dataprep.api.dataset.DataSetMetadata) Test(org.junit.Test)

Example 3 with PreparationMessage

use of org.talend.dataprep.api.preparation.PreparationMessage in project data-prep by Talend.

the class PreparationClientTest method getDetails.

/**
 * Return the details of a preparation at a given (optional) step.
 *
 * @param preparationId the wanted preparation id.
 * @param wantedStepId the optional wanted step id.
 * @return the details of a preparation at a given (optional) step.
 */
public PreparationMessage getDetails(String preparationId, String wantedStepId) {
    final RequestSpecification specs = given();
    if (StringUtils.isNotBlank(wantedStepId)) {
        specs.queryParam("stepId", wantedStepId);
    }
    final Response response = specs.when().get("/preparations/{id}/details", preparationId);
    if (response.getStatusCode() != 200) {
        throw new MockTDPException(response);
    }
    try {
        return mapper.readerFor(PreparationMessage.class).readValue(response.asString());
    } catch (IOException e) {
        throw new TDPException(UNABLE_TO_READ_CONTENT);
    }
}
Also used : Response(com.jayway.restassured.response.Response) TDPException(org.talend.dataprep.exception.TDPException) MockTDPException(org.talend.dataprep.test.MockTDPException) MockTDPException(org.talend.dataprep.test.MockTDPException) RequestSpecification(com.jayway.restassured.specification.RequestSpecification) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) IOException(java.io.IOException)

Example 4 with PreparationMessage

use of org.talend.dataprep.api.preparation.PreparationMessage in project data-prep by Talend.

the class OptimizedExportStrategy method performOptimizedTransform.

private void performOptimizedTransform(ExportParameters parameters, OutputStream outputStream) throws IOException {
    // Initial check
    final OptimizedPreparationInput optimizedPreparationInput = new OptimizedPreparationInput(parameters).invoke();
    if (optimizedPreparationInput == null) {
        throw new IllegalStateException("Unable to use this strategy (call accept() before calling this).");
    }
    final String preparationId = parameters.getPreparationId();
    final String dataSetId = optimizedPreparationInput.getDataSetId();
    final TransformationCacheKey transformationCacheKey = optimizedPreparationInput.getTransformationCacheKey();
    final DataSetMetadata metadata = optimizedPreparationInput.getMetadata();
    final String previousVersion = optimizedPreparationInput.getPreviousVersion();
    final String version = optimizedPreparationInput.getVersion();
    final ExportFormat format = getFormat(parameters.getExportType());
    // Get content from previous step
    try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(contentCache.get(transformationCacheKey), UTF_8))) {
        // Create dataset
        final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
        dataSet.setMetadata(metadata);
        // get the actions to apply (no preparation ==> dataset export ==> no actions)
        final String actions = getActions(preparationId, previousVersion, version);
        final PreparationMessage preparation = getPreparation(preparationId);
        preparation.setSteps(getMatchingSteps(preparation.getSteps(), previousVersion, version));
        LOGGER.debug("Running optimized strategy for preparation {} @ step #{}", preparationId, version);
        // create tee to broadcast to cache + service output
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        dataSetId, // 
        preparationId, // 
        version, // 
        parameters.getExportType(), // 
        parameters.getFrom(), // 
        parameters.getArguments(), // 
        parameters.getFilter());
        LOGGER.debug("Cache key: " + key.getKey());
        LOGGER.debug("Cache key details: " + key.toString());
        try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
            final Configuration configuration = // 
            Configuration.builder().args(// 
            parameters.getArguments()).outFilter(// 
            rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
            format.getName()).actions(// 
            actions).preparation(// 
            preparation).stepId(// 
            version).volume(// 
            Configuration.Volume.SMALL).output(// 
            tee).limit(// 
            limit).build();
            factory.get(configuration).buildExecutable(dataSet, configuration).execute();
            tee.flush();
        } catch (Throwable e) {
            // NOSONAR
            contentCache.evict(key);
            throw e;
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    }
}
Also used : ExportFormat(org.talend.dataprep.format.export.ExportFormat) StringUtils(org.apache.commons.lang.StringUtils) TDPException(org.talend.dataprep.exception.TDPException) TransformationErrorCodes(org.talend.dataprep.exception.error.TransformationErrorCodes) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) ArrayList(java.util.ArrayList) CacheKeyGenerator(org.talend.dataprep.cache.CacheKeyGenerator) TransformationMetadataCacheKey(org.talend.dataprep.cache.TransformationMetadataCacheKey) DataSetMetadata(org.talend.dataprep.api.dataset.DataSetMetadata) TeeOutputStream(org.apache.commons.io.output.TeeOutputStream) ExportUtils(org.talend.dataprep.transformation.service.ExportUtils) DataSet(org.talend.dataprep.api.dataset.DataSet) OutputStream(java.io.OutputStream) Preparation(org.talend.dataprep.api.preparation.Preparation) ExportParameters(org.talend.dataprep.api.export.ExportParameters) Logger(org.slf4j.Logger) BaseExportStrategy(org.talend.dataprep.transformation.service.BaseExportStrategy) JsonParser(com.fasterxml.jackson.core.JsonParser) UTF_8(java.nio.charset.StandardCharsets.UTF_8) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) IOException(java.io.IOException) Step(org.talend.dataprep.api.preparation.Step) InputStreamReader(java.io.InputStreamReader) Collectors(java.util.stream.Collectors) ContentCache(org.talend.dataprep.cache.ContentCache) CSVFormat(org.talend.dataprep.transformation.format.CSVFormat) List(java.util.List) Component(org.springframework.stereotype.Component) TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) InputStream(java.io.InputStream) TeeOutputStream(org.apache.commons.io.output.TeeOutputStream) InputStreamReader(java.io.InputStreamReader) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) DataSet(org.talend.dataprep.api.dataset.DataSet) ExportFormat(org.talend.dataprep.format.export.ExportFormat) DataSetMetadata(org.talend.dataprep.api.dataset.DataSetMetadata) TDPException(org.talend.dataprep.exception.TDPException) IOException(java.io.IOException) TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) TDPException(org.talend.dataprep.exception.TDPException) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) JsonParser(com.fasterxml.jackson.core.JsonParser)

Example 5 with PreparationMessage

use of org.talend.dataprep.api.preparation.PreparationMessage in project data-prep by Talend.

the class PreparationExportStrategy method performPreparation.

public void performPreparation(final ExportParameters parameters, final OutputStream outputStream) {
    final String stepId = parameters.getStepId();
    final String preparationId = parameters.getPreparationId();
    final String formatName = parameters.getExportType();
    final PreparationMessage preparation = getPreparation(preparationId, stepId);
    final String dataSetId = preparation.getDataSetId();
    final ExportFormat format = getFormat(parameters.getExportType());
    // get the dataset content (in an auto-closable block to make sure it is properly closed)
    boolean releasedIdentity = false;
    // Allow get dataset and get dataset metadata access whatever share status is
    securityProxy.asTechnicalUser();
    final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, dataSetId, false, true);
    final DataSetGetMetadata dataSetGetMetadata = applicationContext.getBean(DataSetGetMetadata.class, dataSetId);
    try (InputStream datasetContent = dataSetGet.execute()) {
        try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
            // head is not allowed as step id
            final String version = getCleanStepId(preparation, stepId);
            // Create dataset
            final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
            dataSet.setMetadata(dataSetGetMetadata.execute());
            // All good, can already release identity
            securityProxy.releaseIdentity();
            releasedIdentity = true;
            // get the actions to apply (no preparation ==> dataset export ==> no actions)
            final String actions = getActions(preparationId, version);
            final TransformationCacheKey key = // 
            cacheKeyGenerator.generateContentKey(// 
            dataSetId, // 
            preparationId, // 
            version, // 
            formatName, // 
            parameters.getFrom(), // 
            parameters.getArguments(), // 
            parameters.getFilter());
            LOGGER.debug("Cache key: " + key.getKey());
            LOGGER.debug("Cache key details: " + key.toString());
            try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
                final Configuration configuration = // 
                Configuration.builder().args(// 
                parameters.getArguments()).outFilter(// 
                rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
                format.getName()).actions(// 
                actions).preparation(// 
                preparation).stepId(// 
                version).volume(// 
                Configuration.Volume.SMALL).output(// 
                tee).limit(// 
                limit).build();
                factory.get(configuration).buildExecutable(dataSet, configuration).execute();
                tee.flush();
            } catch (Throwable e) {
                // NOSONAR
                contentCache.evict(key);
                throw e;
            }
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    } finally {
        if (!releasedIdentity) {
            // Release identity in case of error.
            securityProxy.releaseIdentity();
        }
    }
}
Also used : ExportFormat(org.talend.dataprep.format.export.ExportFormat) StringUtils(org.apache.commons.lang.StringUtils) DataSetGet(org.talend.dataprep.command.dataset.DataSetGet) TDPException(org.talend.dataprep.exception.TDPException) TransformationErrorCodes(org.talend.dataprep.exception.error.TransformationErrorCodes) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) CacheKeyGenerator(org.talend.dataprep.cache.CacheKeyGenerator) TeeOutputStream(org.apache.commons.io.output.TeeOutputStream) ExportUtils(org.talend.dataprep.transformation.service.ExportUtils) DataSet(org.talend.dataprep.api.dataset.DataSet) OutputStream(java.io.OutputStream) ExportParameters(org.talend.dataprep.api.export.ExportParameters) Logger(org.slf4j.Logger) HEAD(org.talend.dataprep.api.export.ExportParameters.SourceType.HEAD) BaseExportStrategy(org.talend.dataprep.transformation.service.BaseExportStrategy) JsonParser(com.fasterxml.jackson.core.JsonParser) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) InputStreamReader(java.io.InputStreamReader) ContentCache(org.talend.dataprep.cache.ContentCache) SecurityProxy(org.talend.dataprep.security.SecurityProxy) CSVFormat(org.talend.dataprep.transformation.format.CSVFormat) Component(org.springframework.stereotype.Component) TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) DataSetGetMetadata(org.talend.dataprep.command.dataset.DataSetGetMetadata) InputStream(java.io.InputStream) DataSetGet(org.talend.dataprep.command.dataset.DataSetGet) TeeOutputStream(org.apache.commons.io.output.TeeOutputStream) InputStreamReader(java.io.InputStreamReader) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) DataSet(org.talend.dataprep.api.dataset.DataSet) InputStream(java.io.InputStream) ExportFormat(org.talend.dataprep.format.export.ExportFormat) DataSetGetMetadata(org.talend.dataprep.command.dataset.DataSetGetMetadata) TDPException(org.talend.dataprep.exception.TDPException) TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) TDPException(org.talend.dataprep.exception.TDPException) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) JsonParser(com.fasterxml.jackson.core.JsonParser)

Aggregations

PreparationMessage (org.talend.dataprep.api.preparation.PreparationMessage)9 Test (org.junit.Test)4 EnrichedPreparation (org.talend.dataprep.api.service.api.EnrichedPreparation)4 TDPException (org.talend.dataprep.exception.TDPException)4 Logger (org.slf4j.Logger)3 LoggerFactory (org.slf4j.LoggerFactory)3 Autowired (org.springframework.beans.factory.annotation.Autowired)3 Component (org.springframework.stereotype.Component)3 DataSet (org.talend.dataprep.api.dataset.DataSet)3 DataSetMetadata (org.talend.dataprep.api.dataset.DataSetMetadata)3 Step (org.talend.dataprep.api.preparation.Step)3 CacheKeyGenerator (org.talend.dataprep.cache.CacheKeyGenerator)3 ContentCache (org.talend.dataprep.cache.ContentCache)3 Configuration (org.talend.dataprep.transformation.api.transformer.configuration.Configuration)3 JsonParser (com.fasterxml.jackson.core.JsonParser)2 IOException (java.io.IOException)2 InputStream (java.io.InputStream)2 InputStreamReader (java.io.InputStreamReader)2 OutputStream (java.io.OutputStream)2 UTF_8 (java.nio.charset.StandardCharsets.UTF_8)2