Search in sources :

Example 11 with TransformationCacheKey

use of org.talend.dataprep.cache.TransformationCacheKey in project data-prep by Talend.

the class OptimizedExportStrategyTest method testExecute.

@Test
public void testExecute() throws Exception {
    // Given
    final String datasetId = "1234";
    final String format = "JSON";
    final String preparation = createEmptyPreparationFromDataset(datasetId, "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final Preparation preparationDetails = getPreparation(preparation);
    for (Step step : preparationDetails.getSteps()) {
        try (OutputStream content = contentCache.put(cacheKeyGenerator.generateMetadataKey(preparation, step.id(), HEAD), ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        datasetId, // 
        preparation, // 
        step.id(), // 
        format, // 
        HEAD, // no filter
        "");
        try (OutputStream content = contentCache.put(key, ContentCache.TimeToLive.DEFAULT)) {
            content.write("{\"records\": [{\"0000\": \"a\"}]}".getBytes());
            content.flush();
        }
    }
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setDatasetId(datasetId);
    exportParameters.setExportType(format);
    exportParameters.setFrom(HEAD);
    // Then
    optimizedExportStrategy.execute(exportParameters);
}
Also used : TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) Preparation(org.talend.dataprep.api.preparation.Preparation) ExportParameters(org.talend.dataprep.api.export.ExportParameters) OutputStream(java.io.OutputStream) Step(org.talend.dataprep.api.preparation.Step) Test(org.junit.Test) TransformationServiceBaseTest(org.talend.dataprep.transformation.service.TransformationServiceBaseTest)

Example 12 with TransformationCacheKey

use of org.talend.dataprep.cache.TransformationCacheKey in project data-prep by Talend.

the class PreparationExportStrategy method performPreparation.

public void performPreparation(final ExportParameters parameters, final OutputStream outputStream) {
    final String stepId = parameters.getStepId();
    final String preparationId = parameters.getPreparationId();
    final String formatName = parameters.getExportType();
    final PreparationMessage preparation = getPreparation(preparationId, stepId);
    final String dataSetId = preparation.getDataSetId();
    final ExportFormat format = getFormat(parameters.getExportType());
    // get the dataset content (in an auto-closable block to make sure it is properly closed)
    boolean releasedIdentity = false;
    // Allow get dataset and get dataset metadata access whatever share status is
    securityProxy.asTechnicalUser();
    final DataSetGet dataSetGet = applicationContext.getBean(DataSetGet.class, dataSetId, false, true);
    final DataSetGetMetadata dataSetGetMetadata = applicationContext.getBean(DataSetGetMetadata.class, dataSetId);
    try (InputStream datasetContent = dataSetGet.execute()) {
        try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(datasetContent, UTF_8))) {
            // head is not allowed as step id
            final String version = getCleanStepId(preparation, stepId);
            // Create dataset
            final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
            dataSet.setMetadata(dataSetGetMetadata.execute());
            // All good, can already release identity
            securityProxy.releaseIdentity();
            releasedIdentity = true;
            // get the actions to apply (no preparation ==> dataset export ==> no actions)
            final String actions = getActions(preparationId, version);
            final TransformationCacheKey key = // 
            cacheKeyGenerator.generateContentKey(// 
            dataSetId, // 
            preparationId, // 
            version, // 
            formatName, // 
            parameters.getFrom(), // 
            parameters.getArguments(), // 
            parameters.getFilter());
            LOGGER.debug("Cache key: " + key.getKey());
            LOGGER.debug("Cache key details: " + key.toString());
            try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
                final Configuration configuration = // 
                Configuration.builder().args(// 
                parameters.getArguments()).outFilter(// 
                rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
                format.getName()).actions(// 
                actions).preparation(// 
                preparation).stepId(// 
                version).volume(// 
                Configuration.Volume.SMALL).output(// 
                tee).limit(// 
                limit).build();
                factory.get(configuration).buildExecutable(dataSet, configuration).execute();
                tee.flush();
            } catch (Throwable e) {
                // NOSONAR
                contentCache.evict(key);
                throw e;
            }
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    } finally {
        if (!releasedIdentity) {
            // Release identity in case of error.
            securityProxy.releaseIdentity();
        }
    }
}
Also used : ExportFormat(org.talend.dataprep.format.export.ExportFormat) StringUtils(org.apache.commons.lang.StringUtils) DataSetGet(org.talend.dataprep.command.dataset.DataSetGet) TDPException(org.talend.dataprep.exception.TDPException) TransformationErrorCodes(org.talend.dataprep.exception.error.TransformationErrorCodes) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) CacheKeyGenerator(org.talend.dataprep.cache.CacheKeyGenerator) TeeOutputStream(org.apache.commons.io.output.TeeOutputStream) ExportUtils(org.talend.dataprep.transformation.service.ExportUtils) DataSet(org.talend.dataprep.api.dataset.DataSet) OutputStream(java.io.OutputStream) ExportParameters(org.talend.dataprep.api.export.ExportParameters) Logger(org.slf4j.Logger) HEAD(org.talend.dataprep.api.export.ExportParameters.SourceType.HEAD) BaseExportStrategy(org.talend.dataprep.transformation.service.BaseExportStrategy) JsonParser(com.fasterxml.jackson.core.JsonParser) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) InputStreamReader(java.io.InputStreamReader) ContentCache(org.talend.dataprep.cache.ContentCache) SecurityProxy(org.talend.dataprep.security.SecurityProxy) CSVFormat(org.talend.dataprep.transformation.format.CSVFormat) Component(org.springframework.stereotype.Component) TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) DataSetGetMetadata(org.talend.dataprep.command.dataset.DataSetGetMetadata) InputStream(java.io.InputStream) DataSetGet(org.talend.dataprep.command.dataset.DataSetGet) TeeOutputStream(org.apache.commons.io.output.TeeOutputStream) InputStreamReader(java.io.InputStreamReader) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) DataSet(org.talend.dataprep.api.dataset.DataSet) InputStream(java.io.InputStream) ExportFormat(org.talend.dataprep.format.export.ExportFormat) DataSetGetMetadata(org.talend.dataprep.command.dataset.DataSetGetMetadata) TDPException(org.talend.dataprep.exception.TDPException) TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) TDPException(org.talend.dataprep.exception.TDPException) PreparationMessage(org.talend.dataprep.api.preparation.PreparationMessage) JsonParser(com.fasterxml.jackson.core.JsonParser)

Example 13 with TransformationCacheKey

use of org.talend.dataprep.cache.TransformationCacheKey in project data-prep by Talend.

the class TransformationCacheKeyTest method getKey_should_generate_serialized_key.

@Test
public void getKey_should_generate_serialized_key() throws Exception {
    // given
    final ContentCacheKey key = new TransformationCacheKey("prep1", "dataset1", "JSON", "step1", "param1", HEAD, "user1", "");
    // when
    final String keyStr = key.getKey();
    // then
    assertThat(keyStr, is("transformation_prep1_dataset1_b6aa01425c31e1eed71d0c3cbc7763aad865d1b1"));
}
Also used : TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) ContentCacheKey(org.talend.dataprep.cache.ContentCacheKey) Test(org.junit.Test)

Example 14 with TransformationCacheKey

use of org.talend.dataprep.cache.TransformationCacheKey in project data-prep by Talend.

the class TransformationCacheKeyTest method shouldGenerateSameKey.

@Test
public void shouldGenerateSameKey() throws Exception {
    // given
    final TransformationCacheKey key1 = createTestDefaultKey();
    // same params
    final TransformationCacheKey key2 = createTestDefaultKey();
    // then
    assertEquals(key1.getKey(), key2.getKey());
}
Also used : TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) Test(org.junit.Test)

Example 15 with TransformationCacheKey

use of org.talend.dataprep.cache.TransformationCacheKey in project data-prep by Talend.

the class TransformationCacheKeyTest method getMatcher_should_return_matcher_for_partial_key.

@Test
public void getMatcher_should_return_matcher_for_partial_key() throws Exception {
    // given
    final ContentCacheKey prepKey = new TransformationCacheKey("prep1", null, null, null, null, null, null, "");
    final ContentCacheKey dataSetKey = new TransformationCacheKey(null, "dataset1", null, null, null, null, null, "");
    final ContentCacheKey matchingKey = new TransformationCacheKey("prep1", "dataset1", "JSON", "step1", "param1", HEAD, "user1", "");
    final ContentCacheKey nonMatchingKey = new TransformationCacheKey("prep2", "dataset2", "XLS", "step2", "param2", FILTER, "user2", "");
    // when / then
    assertThat(prepKey.getMatcher().test(matchingKey.getKey()), is(true));
    assertThat(dataSetKey.getMatcher().test(matchingKey.getKey()), is(true));
    assertThat(prepKey.getMatcher().test(nonMatchingKey.getKey()), is(false));
    assertThat(dataSetKey.getMatcher().test(nonMatchingKey.getKey()), is(false));
}
Also used : TransformationCacheKey(org.talend.dataprep.cache.TransformationCacheKey) ContentCacheKey(org.talend.dataprep.cache.ContentCacheKey) Test(org.junit.Test)

Aggregations

TransformationCacheKey (org.talend.dataprep.cache.TransformationCacheKey)16 Test (org.junit.Test)9 Preparation (org.talend.dataprep.api.preparation.Preparation)7 ExportParameters (org.talend.dataprep.api.export.ExportParameters)6 OutputStream (java.io.OutputStream)5 InputStream (java.io.InputStream)4 DataSet (org.talend.dataprep.api.dataset.DataSet)4 TDPException (org.talend.dataprep.exception.TDPException)4 ExportFormat (org.talend.dataprep.format.export.ExportFormat)4 JsonParser (com.fasterxml.jackson.core.JsonParser)3 IOException (java.io.IOException)3 InputStreamReader (java.io.InputStreamReader)3 TeeOutputStream (org.apache.commons.io.output.TeeOutputStream)3 StringUtils (org.apache.commons.lang.StringUtils)3 Autowired (org.springframework.beans.factory.annotation.Autowired)3 Component (org.springframework.stereotype.Component)3 StreamingResponseBody (org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody)3 PreparationMessage (org.talend.dataprep.api.preparation.PreparationMessage)3 Step (org.talend.dataprep.api.preparation.Step)3 CacheKeyGenerator (org.talend.dataprep.cache.CacheKeyGenerator)3