Examples with Action org.talend.dataprep.api.preparation.Action used on opensource projects

Example 21 with Action

use of org.talend.dataprep.api.preparation.Action in project data-prep by Talend.

the class ActionParser method parse.

/**
 * Return the parsed actions ready to be run.
 *
 * @param actions the actions to be parsed as string.
 * @return the parsed actions.
 * @throws IllegalArgumentException if <code>actions</code> is null.
 */
public List<RunnableAction> parse(String actions) {
    if (actions == null) {
        // Actions cannot be null (but can be empty string for no op actions).
        throw new IllegalArgumentException("Actions parameter can not be null.");
    }
    if (StringUtils.isEmpty(actions)) {
        return Collections.emptyList();
    }
    try {
        // Parse action JSON
        final Actions parsedActions = mapper.reader(Actions.class).readValue(actions);
        final List<Action> allActions = parsedActions.getActions();
        // Create closures from parsed actions
        final List<RunnableAction> builtActions = new ArrayList<>(allActions.size() + 1);
        // 
        allActions.stream().filter(// 
        parsedAction -> parsedAction != null && parsedAction.getName() != null).forEach(parsedAction -> {
            String actionNameLowerCase = parsedAction.getName().toLowerCase();
            final ActionDefinition metadata = actionRegistry.get(actionNameLowerCase);
            builtActions.add(factory.create(metadata, parsedAction.getParameters()));
        });
        // all set: wraps everything and return to caller
        return builtActions;
    } catch (TalendRuntimeException tpe) {
        // leave TDPException as is
        throw tpe;
    } catch (Exception e) {
        throw new TalendRuntimeException(BaseErrorCodes.UNABLE_TO_PARSE_JSON, e);
    }
}

Example 22 with Action

use of org.talend.dataprep.api.preparation.Action in project data-prep by Talend.

the class ActionsStaticProfiler method profile.

public ActionsProfile profile(final List<ColumnMetadata> columns, final List<RunnableAction> actions) {
    final Map<Action, ActionDefinition> metadataByAction = getActionMetadataByAction(actions);
    // Compile actions
    final Set<String> originalColumns = columns.stream().map(ColumnMetadata::getId).collect(toSet());
    final Set<String> valueModifiedColumns = new HashSet<>();
    final Set<String> metadataModifiedColumns = new HashSet<>();
    int createColumnActions = 0;
    // Analyze what columns to look at during analysis
    for (Map.Entry<Action, ActionDefinition> entry : metadataByAction.entrySet()) {
        final ActionDefinition actionMetadata = entry.getValue();
        final Action action = entry.getKey();
        Set<ActionDefinition.Behavior> behavior = actionMetadata.getBehavior();
        boolean createColumn = false;
        for (ActionDefinition.Behavior currentBehavior : behavior) {
            switch(currentBehavior) {
                case VALUES_ALL:
                    // All values are going to be changed, and all original columns are going to be modified.
                    valueModifiedColumns.addAll(originalColumns);
                    break;
                case METADATA_CHANGE_TYPE:
                    valueModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    metadataModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    break;
                case VALUES_COLUMN:
                    valueModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    break;
                case VALUES_MULTIPLE_COLUMNS:
                    // Add the action's source column
                    valueModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    // ... then add all column parameter (COLUMN_ID is string, not column)
                    final List<Parameter> parameters = actionMetadata.getParameters(Locale.US);
                    valueModifiedColumns.addAll(// 
                    parameters.stream().filter(// 
                    parameter -> ParameterType.valueOf(parameter.getType().toUpperCase()) == ParameterType.COLUMN).map(// 
                    parameter -> action.getParameters().get(parameter.getName())).collect(Collectors.toList()));
                    break;
                case METADATA_COPY_COLUMNS:
                case METADATA_CREATE_COLUMNS:
                    createColumn = true;
                    break;
                case METADATA_DELETE_COLUMNS:
                case METADATA_CHANGE_NAME:
                    // Do nothing: no need to re-analyze where only name was changed.
                    break;
                default:
                    break;
            }
        }
        if (createColumn || isCreateColumnParameterOn(action)) {
            createColumnActions++;
        }
    }
    // when values are modified, we need to do a full analysis (schema + invalid + stats)
    boolean needFullAnalysis = !valueModifiedColumns.isEmpty() || createColumnActions > 0;
    // when only metadata is modified, we need to re-evaluate the invalids entries
    boolean needOnlyInvalidAnalysis = !needFullAnalysis && !metadataModifiedColumns.isEmpty();
    // only the columns with modified values or new columns need the schema + stats analysis
    SerializablePredicate<ColumnMetadata> filterForFullAnalysis = new FilterForFullAnalysis(originalColumns, valueModifiedColumns);
    // only the columns with metadata change or value changes need to re-evaluate invalids
    Predicate<ColumnMetadata> filterForInvalidAnalysis = new FilterForInvalidAnalysis(filterForFullAnalysis, metadataModifiedColumns);
    return new ActionsProfile(needFullAnalysis, needOnlyInvalidAnalysis, filterForFullAnalysis, filterForInvalidAnalysis, filterForInvalidAnalysis, metadataByAction);
}

Example 23 with Action

use of org.talend.dataprep.api.preparation.Action in project data-prep by Talend.

the class PreparationTest method initialStepWithAppend.

@Test
public void initialStepWithAppend() {
    final String version = versionService.version().getVersionId();
    final List<Action> actions = getSimpleAction("uppercase", "column_name", "lastname");
    final PreparationActions newContent = PreparationActions.ROOT_ACTIONS.append(actions);
    repository.add(newContent);
    final Step s = new Step(Step.ROOT_STEP.id(), newContent.id(), version);
    repository.add(s);
    final Preparation preparation = new Preparation("#5438743", "1234", s.id(), version);
    preparation.setCreationDate(0L);
    repository.add(preparation);
    assertThat(preparation.id(), Is.is("#5438743"));
}

Example 24 with Action

use of org.talend.dataprep.api.preparation.Action in project data-prep by Talend.

the class DiffMetadata method onExecute.

private HttpRequestBase onExecute(final String dataSetId, final String preparationId, final List<Action> actionsToAdd) {
    // original actions (currently applied on the preparation)
    final List<Action> originalActions;
    try {
        originalActions = objectMapper.readerFor(new TypeReference<List<Action>>() {
        }).readValue(getInput());
    } catch (final IOException e) {
        throw new TDPException(UNABLE_TO_READ_PREPARATION, e, withBuilder().put("id", preparationId).build());
    }
    // prepare the preview parameters out of the preparation actions
    final List<PreviewParameters> previewParameters = IntStream.range(0, actionsToAdd.size()).mapToObj((index) -> {
        try {
            // base actions = original actions + actions to add from 0 to index
            final List<Action> previousActionsToAdd = actionsToAdd.subList(0, index);
            final List<Action> baseActions = new ArrayList<>(originalActions);
            baseActions.addAll(previousActionsToAdd);
            // diff actions actions = base actions + the action to add for diff
            final Action singleActionToAdd = actionsToAdd.get(index);
            final List<Action> diffActions = new ArrayList<>(baseActions);
            diffActions.add(singleActionToAdd);
            return new // 
            PreviewParameters(// 
            serializeActions(baseActions), // 
            serializeActions(diffActions), // 
            dataSetId, // 
            null, // 
            null, HEAD);
        } catch (IOException e) {
            throw new TDPException(UNABLE_TO_READ_PREPARATION, e, withBuilder().put("id", preparationId).build());
        }
    }).collect(toList());
    // create the http action to perform
    try {
        final String uri = transformationServiceUrl + "/transform/diff/metadata";
        final HttpPost transformationCall = new HttpPost(uri);
        transformationCall.setEntity(new StringEntity(objectMapper.writer().writeValueAsString(previewParameters), APPLICATION_JSON));
        return transformationCall;
    } catch (JsonProcessingException e) {
        throw new TDPException(CommonErrorCodes.UNEXPECTED_EXCEPTION, e);
    }
}

Example 25 with Action

use of org.talend.dataprep.api.preparation.Action in project data-prep by Talend.

the class PreparationCleanerTest method shouldNotRemovePreparationActions_sharedByMultiplePreparation.

@Test
public void shouldNotRemovePreparationActions_sharedByMultiplePreparation() throws Exception {
    // given
    final String version = "1.2.3";
    final PreparationActions content = new PreparationActions();
    List<Action> actions = new ArrayList<>();
    content.append(actions);
    content.setAppVersion(version);
    when(repository.list(PreparationActions.class)).thenReturn(Stream.of(content));
    when(repository.get(content.id(), PreparationActions.class)).thenReturn(content);
    // 2 preparations, with each, one step that shares the same action
    final Step stepFirstPreparation = new Step(Step.ROOT_STEP.getId(), content.getId(), version);
    final Step stepSecondPreparation = new Step(Step.ROOT_STEP.getId(), content.getId(), version);
    // add the steps to the repository
    when(repository.exist(eq(PersistentStep.class), eq(TqlBuilder.eq("contentId", content.id())))).thenReturn(true);
    when(repository.list(Step.class)).thenReturn(Stream.of(stepFirstPreparation, stepSecondPreparation));
    when(repository.list(eq(PersistentStep.class))).thenReturn(Stream.of(stepFirstPreparation, stepSecondPreparation).map(s -> {
        final PersistentStep persistentStep = new PersistentStep();
        persistentStep.setId(s.id());
        persistentStep.setContent(s.getContent());
        return persistentStep;
    }));
    Preparation firstPreparation = new Preparation("1", null, stepFirstPreparation.getId(), version);
    firstPreparation.setSteps(Collections.singletonList(stepFirstPreparation));
    Preparation secondPreparation = new Preparation("2", null, stepSecondPreparation.getId(), version);
    secondPreparation.setSteps(Collections.singletonList(stepSecondPreparation));
    // add the preparations to the repository
    when(repository.list(Preparation.class)).thenReturn(Stream.of(firstPreparation, secondPreparation));
    // when
    cleaner.removeOrphanSteps();
    // then
    verify(repository, never()).remove(eq(content));
}