Examples with PreparationDTO org.talend.dataprep.api.preparation.PreparationDTO used on opensource projects

Example 36 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PipelineTransformer method buildExecutable.

@Override
public ExecutableTransformer buildExecutable(DataSet input, Configuration configuration) {
    final RowMetadata rowMetadata = input.getMetadata().getRowMetadata();
    final TransformerWriter writer = writerRegistrationService.getWriter(configuration.formatId(), configuration.output(), configuration.getArguments());
    final ConfiguredCacheWriter metadataWriter = new ConfiguredCacheWriter(contentCache, DEFAULT);
    final TransformationMetadataCacheKey metadataKey = cacheKeyGenerator.generateMetadataKey(configuration.getPreparationId(), configuration.stepId(), configuration.getSourceType());
    final PreparationDTO preparation = configuration.getPreparation();
    // function that from a step gives the rowMetadata associated to the previous/parent step
    final Function<String, RowMetadata> stepRowMetadataSupplier = s -> Optional.ofNullable(// 
    s).map(// 
    id -> stepMetadataRepository.get(id)).orElse(null);
    final Pipeline pipeline = Pipeline.Builder.builder().withAnalyzerService(// 
    analyzerService).withActionRegistry(// 
    actionRegistry).withPreparation(// 
    preparation).withActions(// 
    actionParser.parse(configuration.getActions())).withInitialMetadata(rowMetadata, // 
    configuration.volume() == SMALL).withMonitor(// 
    configuration.getMonitor()).withFilter(// 
    configuration.getFilter()).withLimit(// 
    configuration.getLimit()).withFilterOut(// 
    configuration.getOutFilter()).withOutput(// 
    () -> new WriterNode(writer, metadataWriter, metadataKey)).withStatisticsAdapter(// 
    adapter).withStepMetadataSupplier(// 
    stepRowMetadataSupplier).withGlobalStatistics(// 
    configuration.isGlobalStatistics()).allowMetadataChange(// 
    configuration.isAllowMetadataChange()).build();
    // wrap this transformer into an executable transformer
    return new ExecutableTransformer() {

        @Override
        public void execute() {
            final Optional<Span> span = tracer.map(t -> {
                final Span pipelineSpan = t.createSpan("transformer-pipeline");
                pipelineSpan.tag("preparation id", configuration.getPreparationId());
                pipelineSpan.tag("arguments", configuration.getArguments().toString());
                return pipelineSpan;
            });
            try {
                LOGGER.debug("Before execution: {}", pipeline);
                pipeline.execute(input);
                if (preparation != null && JsonFormat.JSON.equals(configuration.formatId())) {
                    final UpdatedStepVisitor visitor = new UpdatedStepVisitor(stepMetadataRepository);
                    pipeline.accept(visitor);
                }
            } finally {
                LOGGER.debug("After execution: {}", pipeline);
                span.ifPresent(s -> {
                    s.tag("execution", pipeline.toString());
                    tracer.ifPresent(t -> t.close(s));
                });
            }
        }

        @Override
        public void signal(Signal signal) {
            pipeline.signal(signal);
        }
    };
}

Example 37 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PreparationExportStrategyTest method shouldUsedVersionedPreparation.

@Test
public void shouldUsedVersionedPreparation() throws IOException {
    // Given
    final ExportParameters parameters = new ExportParameters();
    parameters.setExportType("JSON");
    parameters.setPreparationId("prep-1234");
    parameters.setStepId("step-1234");
    final PreparationDTO preparation = new PreparationDTO();
    preparation.setId("prep-1234");
    preparation.setHeadId("step-1234");
    configurePreparation(preparation, "prep-1234", "step-1234");
    // When
    final StreamingResponseBody body = strategy.execute(parameters);
    body.writeTo(new NullOutputStream());
    // Then
    final ArgumentCaptor<Configuration> captor = ArgumentCaptor.forClass(Configuration.class);
    verify(transformer).buildExecutable(any(), captor.capture());
    assertEquals("prep-1234", captor.getValue().getPreparationId());
    assertEquals("step-1234", captor.getValue().getPreparation().getHeadId());
}

Example 38 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PreparationExportStrategyTest method shouldUsedHeadPreparation.

@Test
public void shouldUsedHeadPreparation() throws IOException {
    // Given
    final ExportParameters parameters = new ExportParameters();
    parameters.setExportType("JSON");
    parameters.setPreparationId("prep-1234");
    parameters.setStepId("head");
    final PreparationDTO preparation = new PreparationDTO();
    preparation.getSteps().add(Step.ROOT_STEP.id());
    preparation.setId("prep-1234");
    preparation.setHeadId("head");
    configurePreparation(preparation, "prep-1234", "head");
    // When
    final StreamingResponseBody body = strategy.execute(parameters);
    body.writeTo(new NullOutputStream());
    // Then
    final ArgumentCaptor<Configuration> captor = ArgumentCaptor.forClass(Configuration.class);
    verify(transformer).buildExecutable(any(), captor.capture());
    assertEquals("prep-1234", captor.getValue().getPreparationId());
    assertEquals("head", captor.getValue().getPreparation().getHeadId());
}

Example 39 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PrepHasStepCondition method apply.

@Override
public boolean apply(Object... args) {
    // check pre-condition
    Validate.notNull(args);
    Validate.isTrue(args.length == 2);
    Validate.isInstanceOf(String.class, args[0]);
    Validate.isInstanceOf(String.class, args[1]);
    String preparationId = (String) args[0];
    String headId = (String) args[1];
    PreparationDTO prep = exportParametersUtil.getPreparation(preparationId, headId);
    return prep.getSteps().size() > 1;
}

Example 40 with PreparationDTO

use of org.talend.dataprep.api.preparation.PreparationDTO in project data-prep by Talend.

the class PreparationAPI method previewUpdate.

// @formatter:off
@RequestMapping(value = "/api/preparations/preview/update", method = POST, consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE)
@ApiOperation(value = "Get a preview diff between the same step of the same preparation but with one step update.")
public StreamingResponseBody previewUpdate(@RequestBody final PreviewUpdateParameters input) {
    // @formatter:on
    // get preparation details
    final PreparationDTO preparation = internalGetPreparation(input.getPreparationId());
    final List<Action> actions = internalGetActions(preparation.getId());
    final HystrixCommand<InputStream> transformation = getCommand(PreviewUpdate.class, input, preparation, actions);
    return executePreviewCommand(transformation);
}