Examples with Step org.talend.dataprep.api.preparation.Step used on opensource projects

Example 6 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class OptimizedExportStrategyTest method testAcceptOK.

@Test
public void testAcceptOK() throws Exception {
    // Given
    final String datasetId = "1234";
    final String format = "";
    final String preparation = createEmptyPreparationFromDataset(datasetId, "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final Preparation preparationDetails = getPreparation(preparation);
    for (Step step : preparationDetails.getSteps()) {
        try (OutputStream content = contentCache.put(cacheKeyGenerator.generateMetadataKey(preparation, step.id(), HEAD), ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        datasetId, // 
        preparation, // 
        step.id(), // 
        format, // 
        HEAD, // 
        "");
        try (OutputStream content = contentCache.put(key, ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
    }
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setDatasetId(datasetId);
    exportParameters.setExportType(format);
    exportParameters.setFrom(HEAD);
    // Then
    assertTrue(optimizedExportStrategy.accept(exportParameters));
}

Example 7 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class OptimizedExportStrategy method performOptimizedTransform.

private void performOptimizedTransform(ExportParameters parameters, OutputStream outputStream) throws IOException {
    // Initial check
    final OptimizedPreparationInput optimizedPreparationInput = new OptimizedPreparationInput(parameters).invoke();
    if (optimizedPreparationInput == null) {
        throw new IllegalStateException("Unable to use this strategy (call accept() before calling this).");
    }
    final String preparationId = parameters.getPreparationId();
    final String dataSetId = optimizedPreparationInput.getDataSetId();
    final TransformationCacheKey transformationCacheKey = optimizedPreparationInput.getTransformationCacheKey();
    final DataSetMetadata metadata = optimizedPreparationInput.getMetadata();
    final String previousVersion = optimizedPreparationInput.getPreviousVersion();
    final String version = optimizedPreparationInput.getVersion();
    final ExportFormat format = getFormat(parameters.getExportType());
    // Get content from previous step
    try (JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(contentCache.get(transformationCacheKey), UTF_8))) {
        // Create dataset
        final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
        dataSet.setMetadata(metadata);
        // get the actions to apply (no preparation ==> dataset export ==> no actions)
        final String actions = getActions(preparationId, previousVersion, version);
        final PreparationMessage preparation = getPreparation(preparationId);
        preparation.setSteps(getMatchingSteps(preparation.getSteps(), previousVersion, version));
        LOGGER.debug("Running optimized strategy for preparation {} @ step #{}", preparationId, version);
        // create tee to broadcast to cache + service output
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        dataSetId, // 
        preparationId, // 
        version, // 
        parameters.getExportType(), // 
        parameters.getFrom(), // 
        parameters.getArguments(), // 
        parameters.getFilter());
        LOGGER.debug("Cache key: " + key.getKey());
        LOGGER.debug("Cache key details: " + key.toString());
        try (final TeeOutputStream tee = new TeeOutputStream(outputStream, contentCache.put(key, ContentCache.TimeToLive.DEFAULT))) {
            final Configuration configuration = // 
            Configuration.builder().args(// 
            parameters.getArguments()).outFilter(// 
            rm -> filterService.build(parameters.getFilter(), rm)).sourceType(parameters.getFrom()).format(// 
            format.getName()).actions(// 
            actions).preparation(// 
            preparation).stepId(// 
            version).volume(// 
            Configuration.Volume.SMALL).output(// 
            tee).limit(// 
            limit).build();
            factory.get(configuration).buildExecutable(dataSet, configuration).execute();
            tee.flush();
        } catch (Throwable e) {
            // NOSONAR
            contentCache.evict(key);
            throw e;
        }
    } catch (TDPException e) {
        throw e;
    } catch (Exception e) {
        throw new TDPException(TransformationErrorCodes.UNABLE_TO_TRANSFORM_DATASET, e);
    }
}

Example 8 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class PreparationCleanerTest method removeOrphanSteps_should_remove_orphan_step_content.

@Test
public void removeOrphanSteps_should_remove_orphan_step_content() {
    // given
    final String version = "1.2.3";
    final PreparationActions content = new PreparationActions();
    content.setAppVersion(version);
    when(repository.list(PreparationActions.class)).thenReturn(Stream.of(content));
    when(repository.get(content.id(), PreparationActions.class)).thenReturn(content);
    final Step step = new Step(Step.ROOT_STEP.id(), content.id(), version);
    when(repository.list(eq(Step.class))).thenReturn(Stream.of(step));
    when(repository.list(eq(PersistentStep.class))).thenReturn(Stream.of(step).map(s -> {
        final PersistentStep persistentStep = new PersistentStep();
        persistentStep.setId(s.id());
        persistentStep.setContent(s.getContent());
        return persistentStep;
    }));
    when(repository.list(eq(Preparation.class))).thenReturn(Stream.empty());
    // when
    cleaner.removeOrphanSteps();
    // then
    verify(repository, times(1)).remove(eq(content));
}

Example 9 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class PreparationCleanerTest method removeOrphanSteps_should_not_remove_step_that_still_belongs_to_a_preparation.

@Test
public void removeOrphanSteps_should_not_remove_step_that_still_belongs_to_a_preparation() {
    // given
    final String version = "1.2.3";
    final Step firstStep = new Step(Step.ROOT_STEP.id(), "first", version);
    final Step secondStep = new Step(firstStep.id(), "second", version);
    final Step thirdStep = new Step(secondStep.id(), "third", version);
    when(repository.list(eq(Step.class))).thenReturn(Stream.of(firstStep, secondStep, thirdStep));
    when(repository.list(eq(PersistentStep.class))).thenReturn(Stream.of(firstStep, secondStep, thirdStep).map(s -> {
        final PersistentStep persistentStep = new PersistentStep();
        persistentStep.setId(s.id());
        persistentStep.setContent(s.getContent());
        return persistentStep;
    }));
    final Preparation firstPreparation = new Preparation("#458", "1", firstStep.id(), version);
    firstPreparation.setSteps(Collections.singletonList(firstStep));
    final Preparation secondPreparation = new Preparation("#5428", "2", thirdStep.id(), version);
    secondPreparation.setSteps(Arrays.asList(thirdStep, secondStep, firstStep));
    when(repository.list(eq(Preparation.class))).thenReturn(Stream.of(firstPreparation, secondPreparation));
    // when
    cleaner.removeOrphanSteps();
    // then
    verify(repository, never()).remove(eq(firstStep));
    verify(repository, never()).remove(eq(secondStep));
    verify(repository, never()).remove(eq(thirdStep));
}

Example 10 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class SortAndOrderHelperTest method createPreparation.

private Preparation createPreparation(String name, String author, long creation, long modification, long size, String dsId) {
    Preparation firstPrep = new Preparation();
    firstPrep.setDataSetId(dsId);
    firstPrep.setName(name);
    firstPrep.setAuthor(new Owner("1234", author, "").getDisplayName());
    firstPrep.setCreationDate(creation);
    firstPrep.setLastModificationDate(modification);
    List<Step> steps = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        steps.add(null);
    }
    firstPrep.setSteps(steps);
    return firstPrep;
}