Examples with Step org.talend.dataprep.api.preparation.Step used on opensource projects

Example 26 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class SortAndOrderHelperTest method createUserPreparation.

private Preparation createUserPreparation(String name, String author, long creation, long modification, long size, String dsId) {
    UserPreparation firstPrep = new UserPreparation();
    firstPrep.setDataSetId(dsId);
    firstPrep.setName(name);
    firstPrep.setAuthor("1234");
    firstPrep.setOwner(new Owner("1234", author, ""));
    firstPrep.setCreationDate(creation);
    firstPrep.setLastModificationDate(modification);
    List<Step> steps = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        steps.add(null);
    }
    firstPrep.setSteps(steps);
    return firstPrep;
}

Example 27 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class PreparationCleaner method removeCurrentOrphanSteps.

/**
 * Remove all orphan steps in preparation repository.
 */
public void removeCurrentOrphanSteps() {
    securityProxy.asTechnicalUser();
    try {
        getCurrentOrphanSteps().forEach(step -> {
            // Remove step
            final Step stepToRemove = new Step();
            stepToRemove.setId(step.getId());
            repository.remove(stepToRemove);
            // Remove actions linked to step
            // if this step re-use an existing actions we don't delete the actions
            boolean criterion = repository.exist(PersistentStep.class, eq("contentId", step.getContent()));
            if (criterion) {
                LOGGER.debug("Don't removing step content {} it still used by another step.", step.getContent());
            } else {
                LOGGER.debug("Removing step content {}.", step.getContent());
                final PreparationActions preparationActionsToRemove = new PreparationActions();
                preparationActionsToRemove.setId(step.getContent());
                repository.remove(preparationActionsToRemove);
            }
            // Remove metadata linked to step
            final StepRowMetadata stepRowMetadataToRemove = new StepRowMetadata();
            stepRowMetadataToRemove.setId(stepToRemove.getRowMetadata());
            repository.remove(stepRowMetadataToRemove);
        });
    } finally {
        securityProxy.releaseIdentity();
    }
}

Example 28 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class PipelineTransformer method buildExecutable.

@Override
public ExecutableTransformer buildExecutable(DataSet input, Configuration configuration) {
    final RowMetadata rowMetadata = input.getMetadata().getRowMetadata();
    // prepare the fallback row metadata
    RowMetadata fallBackRowMetadata = transformationRowMetadataUtils.getMatchingEmptyRowMetadata(rowMetadata);
    final TransformerWriter writer = writerRegistrationService.getWriter(configuration.formatId(), configuration.output(), configuration.getArguments());
    final ConfiguredCacheWriter metadataWriter = new ConfiguredCacheWriter(contentCache, DEFAULT);
    final TransformationMetadataCacheKey metadataKey = cacheKeyGenerator.generateMetadataKey(configuration.getPreparationId(), configuration.stepId(), configuration.getSourceType());
    final PreparationMessage preparation = configuration.getPreparation();
    // function that from a step gives the rowMetadata associated to the previous/parent step
    final Function<Step, RowMetadata> previousStepRowMetadataSupplier = s -> // 
    Optional.ofNullable(s.getParent()).map(// 
    id -> preparationUpdater.get(id)).orElse(null);
    final Pipeline pipeline = // 
    Pipeline.Builder.builder().withAnalyzerService(// 
    analyzerService).withActionRegistry(// 
    actionRegistry).withPreparation(// 
    preparation).withActions(// 
    actionParser.parse(configuration.getActions())).withInitialMetadata(rowMetadata, // 
    configuration.volume() == SMALL).withMonitor(// 
    configuration.getMonitor()).withFilter(// 
    configuration.getFilter()).withLimit(// 
    configuration.getLimit()).withFilterOut(// 
    configuration.getOutFilter()).withOutput(// 
    () -> new WriterNode(writer, metadataWriter, metadataKey, fallBackRowMetadata)).withStatisticsAdapter(// 
    adapter).withStepMetadataSupplier(// 
    previousStepRowMetadataSupplier).withGlobalStatistics(// 
    configuration.isGlobalStatistics()).allowMetadataChange(// 
    configuration.isAllowMetadataChange()).build();
    // wrap this transformer into an executable transformer
    return new ExecutableTransformer() {

        @Override
        public void execute() {
            try {
                LOGGER.debug("Before transformation: {}", pipeline);
                pipeline.execute(input);
            } finally {
                LOGGER.debug("After transformation: {}", pipeline);
            }
            if (preparation != null) {
                final UpdatedStepVisitor visitor = new UpdatedStepVisitor(preparationUpdater);
                pipeline.accept(visitor);
            }
        }

        @Override
        public void signal(Signal signal) {
            pipeline.signal(signal);
        }
    };
}