Examples with RunnableAction org.talend.dataprep.transformation.actions.common.RunnableAction used on opensource projects

Example 26 with RunnableAction

use of org.talend.dataprep.transformation.actions.common.RunnableAction in project data-prep by Talend.

the class ActionParser method parse.

/**
 * Return the parsed actions ready to be run.
 *
 * @param actions the actions to be parsed as string.
 * @return the parsed actions.
 * @throws IllegalArgumentException if <code>actions</code> is null.
 */
public List<RunnableAction> parse(String actions) {
    if (actions == null) {
        // Actions cannot be null (but can be empty string for no op actions).
        throw new IllegalArgumentException("Actions parameter can not be null.");
    }
    if (StringUtils.isEmpty(actions)) {
        return Collections.emptyList();
    }
    try {
        // Parse action JSON
        final Actions parsedActions = mapper.reader(Actions.class).readValue(actions);
        final List<Action> allActions = parsedActions.getActions();
        // Create closures from parsed actions
        final List<RunnableAction> builtActions = new ArrayList<>(allActions.size() + 1);
        // 
        allActions.stream().filter(// 
        parsedAction -> parsedAction != null && parsedAction.getName() != null).forEach(parsedAction -> {
            String actionNameLowerCase = parsedAction.getName().toLowerCase();
            final ActionDefinition metadata = actionRegistry.get(actionNameLowerCase);
            builtActions.add(factory.create(metadata, parsedAction.getParameters()));
        });
        // all set: wraps everything and return to caller
        return builtActions;
    } catch (TalendRuntimeException tpe) {
        // leave TDPException as is
        throw tpe;
    } catch (Exception e) {
        throw new TalendRuntimeException(BaseErrorCodes.UNABLE_TO_PARSE_JSON, e);
    }
}

Example 27 with RunnableAction

use of org.talend.dataprep.transformation.actions.common.RunnableAction in project data-prep by Talend.

the class ActionsStaticProfiler method profile.

public ActionsProfile profile(final List<ColumnMetadata> columns, final List<RunnableAction> actions) {
    final Map<Action, ActionDefinition> metadataByAction = getActionMetadataByAction(actions);
    // Compile actions
    final Set<String> originalColumns = columns.stream().map(ColumnMetadata::getId).collect(toSet());
    final Set<String> valueModifiedColumns = new HashSet<>();
    final Set<String> metadataModifiedColumns = new HashSet<>();
    int createColumnActions = 0;
    // Analyze what columns to look at during analysis
    for (Map.Entry<Action, ActionDefinition> entry : metadataByAction.entrySet()) {
        final ActionDefinition actionMetadata = entry.getValue();
        final Action action = entry.getKey();
        Set<ActionDefinition.Behavior> behavior = actionMetadata.getBehavior();
        boolean createColumn = false;
        for (ActionDefinition.Behavior currentBehavior : behavior) {
            switch(currentBehavior) {
                case VALUES_ALL:
                    // All values are going to be changed, and all original columns are going to be modified.
                    valueModifiedColumns.addAll(originalColumns);
                    break;
                case METADATA_CHANGE_TYPE:
                    valueModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    metadataModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    break;
                case VALUES_COLUMN:
                    valueModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    break;
                case VALUES_MULTIPLE_COLUMNS:
                    // Add the action's source column
                    valueModifiedColumns.add(action.getParameters().get(COLUMN_ID.getKey()));
                    // ... then add all column parameter (COLUMN_ID is string, not column)
                    final List<Parameter> parameters = actionMetadata.getParameters(Locale.US);
                    valueModifiedColumns.addAll(// 
                    parameters.stream().filter(// 
                    parameter -> ParameterType.valueOf(parameter.getType().toUpperCase()) == ParameterType.COLUMN).map(// 
                    parameter -> action.getParameters().get(parameter.getName())).collect(Collectors.toList()));
                    break;
                case METADATA_COPY_COLUMNS:
                case METADATA_CREATE_COLUMNS:
                    createColumn = true;
                    break;
                case METADATA_DELETE_COLUMNS:
                case METADATA_CHANGE_NAME:
                    // Do nothing: no need to re-analyze where only name was changed.
                    break;
                default:
                    break;
            }
        }
        if (createColumn || isCreateColumnParameterOn(action)) {
            createColumnActions++;
        }
    }
    // when values are modified, we need to do a full analysis (schema + invalid + stats)
    boolean needFullAnalysis = !valueModifiedColumns.isEmpty() || createColumnActions > 0;
    // when only metadata is modified, we need to re-evaluate the invalids entries
    boolean needOnlyInvalidAnalysis = !needFullAnalysis && !metadataModifiedColumns.isEmpty();
    // only the columns with modified values or new columns need the schema + stats analysis
    SerializablePredicate<ColumnMetadata> filterForFullAnalysis = new FilterForFullAnalysis(originalColumns, valueModifiedColumns);
    // only the columns with metadata change or value changes need to re-evaluate invalids
    Predicate<ColumnMetadata> filterForInvalidAnalysis = new FilterForInvalidAnalysis(filterForFullAnalysis, metadataModifiedColumns);
    return new ActionsProfile(needFullAnalysis, needOnlyInvalidAnalysis, filterForFullAnalysis, filterForInvalidAnalysis, filterForInvalidAnalysis, metadataByAction);
}

Example 28 with RunnableAction

use of org.talend.dataprep.transformation.actions.common.RunnableAction in project data-prep by Talend.

the class MetadataChangesOnActionsGenerator method compileActionsOnMetadata.

/**
 * Compile the given actions, hence updating the given row metadata and return the later.
 *
 * @param actions the actions to compile.
 * @param startingRowMetadata the row metadata to start from.
 * @return the updated row metadata from the actions compilation.
 */
private RowMetadata compileActionsOnMetadata(List<RunnableAction> actions, final RowMetadata startingRowMetadata) {
    final RowMetadata updatedRowMetadata = startingRowMetadata.clone();
    TransformationContext transformationContext = new TransformationContext();
    // compile every action within the transformation context
    for (RunnableAction action : actions) {
        final DataSetRowAction rowAction = action.getRowAction();
        final ActionContext actionContext = transformationContext.create(rowAction, updatedRowMetadata);
        rowAction.compile(actionContext);
    }
    // the cleanup is REALLY important (as it can close http connection in case of a lookup for instance)
    transformationContext.cleanup();
    return updatedRowMetadata;
}

Example 29 with RunnableAction

use of org.talend.dataprep.transformation.actions.common.RunnableAction in project data-prep by Talend.

the class MakeLineHeaderTest method should_keep_header_after_modifications_with_concat.

@Test
public void should_keep_header_after_modifications_with_concat() {
    // given
    Long rowId = 0L;
    // row 1
    Map<String, String> rowContent = new HashMap<>();
    rowContent.put("0000", "David");
    rowContent.put("0001", "Bowie");
    final DataSetRow row1 = new DataSetRow(rowContent);
    row1.setTdpId(rowId++);
    // row 2
    rowContent = new HashMap<>();
    rowContent.put("0000", "John");
    rowContent.put("0001", "Lennon");
    final DataSetRow row2 = new DataSetRow(rowContent);
    row2.setTdpId(rowId++);
    // row 3
    rowContent = new HashMap<>();
    rowContent.put("0000", "Johnny");
    rowContent.put("0001", "Lennon");
    final DataSetRow row3 = new DataSetRow(rowContent);
    row3.setTdpId(rowId++);
    // when
    final Map<String, String> makeHeaderParameters = new HashMap<>();
    makeHeaderParameters.put(ImplicitParameters.SCOPE.getKey().toLowerCase(), "line");
    makeHeaderParameters.put("row_id", row2.getTdpId().toString());
    final RunnableAction makeHeader = factory.create(action, makeHeaderParameters);
    final Map<String, String> concatColumnParameters = new HashMap<>();
    concatColumnParameters.put(ImplicitParameters.SCOPE.getKey().toLowerCase(), "column");
    concatColumnParameters.put(ImplicitParameters.COLUMN_ID.getKey().toLowerCase(), "0000");
    concatColumnParameters.put(OtherColumnParameters.MODE_PARAMETER, "other_column_mode");
    concatColumnParameters.put(OtherColumnParameters.SELECTED_COLUMN_PARAMETER, "0001");
    concatColumnParameters.put(ActionsUtils.CREATE_NEW_COLUMN, "true");
    final RunnableAction concat = factory.create(new Concat(), concatColumnParameters);
    ActionTestWorkbench.test(Arrays.asList(row1, row2, row3), actionRegistry, makeHeader, concat);
    // then
    // assertEquals(3, row3.getRowMetadata().getColumns().size());
    assertEquals("John", row1.getRowMetadata().getById("0000").getName());
    assertEquals("Lennon", row1.getRowMetadata().getById("0001").getName());
    assertEquals("John", row2.getRowMetadata().getById("0000").getName());
    assertEquals("Lennon", row2.getRowMetadata().getById("0001").getName());
    assertEquals("John", row3.getRowMetadata().getById("0000").getName());
    assertEquals("Lennon", row3.getRowMetadata().getById("0001").getName());
}

Example 30 with RunnableAction

use of org.talend.dataprep.transformation.actions.common.RunnableAction in project data-prep by Talend.

the class ActionParserTest method empty_string_should_return_noop_actions.

@Test
public void empty_string_should_return_noop_actions() {
    // given
    DataSetRow actualRow = getDataSetRow();
    DataSetRow expectedRow = actualRow.clone();
    RowMetadata expectedMetadata = getRowMetadata();
    List<RunnableAction> actualActions = actionParser.parse("");
    // when
    final RunnableAction[] actions = actualActions.toArray(new RunnableAction[actualActions.size()]);
    ActionTestWorkbench.test(actualRow, actionRegistry, actions);
    // then
    assertEquals(expectedRow, actualRow);
    assertEquals(expectedMetadata, actualRow.getRowMetadata());
}