Examples with Step org.talend.dataprep.api.preparation.Step used on opensource projects

Example 11 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class PreparationTest method stepsWithAppend.

@Test
public void stepsWithAppend() {
    final String version = versionService.version().getVersionId();
    final List<Action> actions = getSimpleAction("uppercase", "column_name", "lastname");
    final PreparationActions newContent1 = PreparationActions.ROOT_ACTIONS.append(actions);
    repository.add(newContent1);
    final PreparationActions newContent2 = newContent1.append(actions);
    repository.add(newContent2);
    // Steps
    final Step s1 = new Step(Step.ROOT_STEP.id(), newContent1.id(), version);
    repository.add(s1);
    final Step s2 = new Step(s1.id(), newContent2.id(), version);
    repository.add(s2);
    // Preparation
    final Preparation preparation = new Preparation("#54258728", "1234", s2.id(), version);
    preparation.setCreationDate(0L);
    repository.add(preparation);
    assertThat(preparation.id(), Is.is("#54258728"));
}

Example 12 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class PreparationTest method initialStep.

@Test
public void initialStep() {
    final String version = versionService.version().getVersionId();
    final List<Action> actions = getSimpleAction("uppercase", "column_name", "lastname");
    final PreparationActions newContent = new PreparationActions(actions, version);
    repository.add(newContent);
    final Step s = new Step(Step.ROOT_STEP.id(), newContent.id(), version);
    repository.add(s);
    Preparation preparation = new Preparation("#48368", "1234", s.id(), version);
    preparation.setCreationDate(0L);
    repository.add(preparation);
    assertThat(preparation.id(), Is.is("#48368"));
}

Example 13 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class ToPEPersistentIdentifiable method run.

@Override
public void run() {
    LOGGER.debug("starting upgrade from {} to {}.", Step.class, PersistentStep.class);
    final AtomicLong counter = new AtomicLong(0L);
    fileSystemPreparationRepository.list(Step.class).forEach(s -> {
        fileSystemPreparationRepository.remove(s);
        PersistentStep persistentStep = turnToPersistentStep(s);
        preparationRepository.add(persistentStep);
        LOGGER.debug("step {} updated to {}", s, persistentStep);
        counter.incrementAndGet();
    });
    LOGGER.info("Upgrade from {} to {} done, {} steps processed.", Step.class, PersistentStep.class, counter.get());
    LOGGER.debug("starting upgrade from {} to {}.", Preparation.class, PersistentPreparation.class);
    final Stream<Preparation> preparations = fileSystemPreparationRepository.list(Preparation.class);
    preparations.forEach(p -> {
        fileSystemPreparationRepository.remove(p);
        PersistentPreparation persistentPreparation = turnToPersistentPreparation(p);
        preparationRepository.add(persistentPreparation);
    });
    LOGGER.info("Upgrade from {} to {} done.", Preparation.class, PersistentPreparation.class);
    LOGGER.info("Migration of step ids in preparation...");
    final Stream<PersistentPreparation> persistentPreparations = preparationRepository.list(PersistentPreparation.class);
    persistentPreparations.forEach(p -> {
        LOGGER.info("Migration of preparation #{}", p.getId());
        final List<String> stepsIds = preparationUtils.listStepsIds(p.getHeadId(), preparationRepository);
        p.setSteps(stepsIds);
        final DataSetMetadata metadata = dataSetMetadataRepository.get(p.getDataSetId());
        if (metadata != null) {
            LOGGER.info("Set metadata {} in preparation {}.", p.getDataSetId(), p.getId());
            p.setRowMetadata(metadata.getRowMetadata());
        } else {
            LOGGER.info("Metadata {} not found for preparation {}.", p.getDataSetId(), p.getId());
            p.setRowMetadata(new RowMetadata());
        }
        preparationRepository.add(p);
        LOGGER.info("Migration of preparation #{} done ({} steps)", p.getId(), stepsIds.size());
    });
    LOGGER.info("Migration of step ids in preparation done.");
}

Example 14 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class OptimizedExportStrategyTest method testAcceptKO_withMetadataCacheNoContentCache.

@Test
public void testAcceptKO_withMetadataCacheNoContentCache() throws Exception {
    // Given
    final String preparation = createEmptyPreparationFromDataset("1234", "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final Preparation preparationDetails = getPreparation(preparation);
    for (Step step : preparationDetails.getSteps()) {
        try (OutputStream content = contentCache.put(cacheKeyGenerator.generateMetadataKey(preparation, step.id(), HEAD), ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
    }
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setFrom(HEAD);
    // Then
    assertFalse(optimizedExportStrategy.accept(exportParameters));
}

Example 15 with Step

use of org.talend.dataprep.api.preparation.Step in project data-prep by Talend.

the class OptimizedExportStrategyTest method testExecute.

@Test
public void testExecute() throws Exception {
    // Given
    final String datasetId = "1234";
    final String format = "JSON";
    final String preparation = createEmptyPreparationFromDataset(datasetId, "test");
    applyAction(preparation, "[{}]");
    applyAction(preparation, "[{}]");
    final Preparation preparationDetails = getPreparation(preparation);
    for (Step step : preparationDetails.getSteps()) {
        try (OutputStream content = contentCache.put(cacheKeyGenerator.generateMetadataKey(preparation, step.id(), HEAD), ContentCache.TimeToLive.DEFAULT)) {
            content.write("{}".getBytes());
            content.flush();
        }
        final TransformationCacheKey key = // 
        cacheKeyGenerator.generateContentKey(// 
        datasetId, // 
        preparation, // 
        step.id(), // 
        format, // 
        HEAD, // no filter
        "");
        try (OutputStream content = contentCache.put(key, ContentCache.TimeToLive.DEFAULT)) {
            content.write("{\"records\": [{\"0000\": \"a\"}]}".getBytes());
            content.flush();
        }
    }
    ExportParameters exportParameters = new ExportParameters();
    exportParameters.setPreparationId(preparation);
    exportParameters.setDatasetId(datasetId);
    exportParameters.setExportType(format);
    exportParameters.setFrom(HEAD);
    // Then
    optimizedExportStrategy.execute(exportParameters);
}