Examples with MappingIODefinition org.pentaho.di.trans.steps.mapping.MappingIODefinition used on opensource projects

Example 6 with MappingIODefinition

use of org.pentaho.di.trans.steps.mapping.MappingIODefinition in project pentaho-metaverse by pentaho.

the class BaseMappingAnalyzer method createLinks.

private void createLinks(final TransMeta subTransMeta, final StepWithMappingMeta meta) {
    boolean renameFields = shouldRenameFields(meta);
    final Map<String, String> fieldRenames = new HashMap();
    // process input mappings
    for (final MappingIODefinition inputMapping : meta.getInputMappings()) {
        final Vertex inputSourceVertex = getInputSourceStepVertex(parentTransMeta, inputMapping);
        final Vertex inputTargetVertex = getInputTargetStepVertex(subTransMeta, inputMapping);
        final String inputTargetName = inputTargetVertex.getProperty(DictionaryConst.PROPERTY_NAME);
        final List<MappingValueRename> renames = inputMapping.getValueRenames();
        for (final MappingValueRename rename : renames) {
            // This is deliberately target > source, since we are going to be looking up the target to rename back to the
            // source
            fieldRenames.put(rename.getTargetValueName(), rename.getSourceValueName());
        }
        // traverse output fields of the input source step
        // for each field, if a rename exists, create a "derives" link from the field to the input target field with
        // the name defined in the "Fieldname to mapping input step" column; otherwise create a "derives" link to a
        // field with the same name
        final Iterator<Vertex> inputSourceOutputFields = inputSourceVertex.getVertices(Direction.OUT, DictionaryConst.LINK_OUTPUTS).iterator();
        while (inputSourceOutputFields.hasNext()) {
            final Vertex inputSourceOutputField = inputSourceOutputFields.next();
            final String inputSourceOutputFieldName = inputSourceOutputField.getProperty(DictionaryConst.PROPERTY_NAME);
            // is there a rename for this field?
            final MappingValueRename renameMapping = inputMapping.getValueRenames().stream().filter(rename -> inputSourceOutputFieldName.equals(rename.getSourceValueName())).findAny().orElse(null);
            // if there is no rename for this field, we look for a field with the same name, otherwise we look for a field
            // that is the target of the rename mapping ( defined within the "Fieldname to mapping input step" column);
            // we look for this field within the sub-transformation and within the context of the input target step
            final String derivedFieldName = renameMapping == null ? inputSourceOutputFieldName : renameMapping.getTargetValueName();
            final Vertex derivedField = findFieldVertex(subTransMeta, inputTargetName, derivedFieldName);
            // add an "inputs" link from the field of the input source step to the input target step
            metaverseBuilder.addLink(inputSourceOutputField, DictionaryConst.LINK_INPUTS, inputTargetVertex);
            if (derivedField == null) {
                LOGGER.debug(Messages.getString("WARN.TargetInputFieldNotFound", meta.getName(), inputSourceVertex.getProperty(DictionaryConst.PROPERTY_NAME), inputSourceOutputField.getProperty(DictionaryConst.PROPERTY_NAME), subTransMeta.getName(), inputTargetName));
            } else {
                // add a "derives" link from the field of the input source step to the output field of the input target step
                metaverseBuilder.addLink(inputSourceOutputField, DictionaryConst.LINK_DERIVES, derivedField);
            }
        }
    }
    // process output mappings
    for (final MappingIODefinition outputMapping : meta.getOutputMappings()) {
        final Vertex outputSourceVertex = getOutputSourceStepVertex(subTransMeta, outputMapping);
        final List<Vertex> outputTargetVertices = getOutputTargetStepVertices(parentTransMeta, outputMapping);
        // traverse output fields of the output source step
        // for each field, if a rename exists, create a "derives" link from the field to the input target field with
        // the name defined in the "Fieldname to mapping input step" column; otherwise create a "derives" link to a
        // field with the same name
        final Iterator<Vertex> outputSourceFields = outputSourceVertex.getVertices(Direction.OUT, DictionaryConst.LINK_OUTPUTS).iterator();
        while (outputSourceFields.hasNext()) {
            final Vertex outputSourceField = outputSourceFields.next();
            final String outputSourceFieldName = outputSourceField.getProperty(DictionaryConst.PROPERTY_NAME);
            String derivedFieldName = outputSourceFieldName;
            // is there a rename mapping for this field?
            final MappingValueRename renameMapping = outputMapping.getValueRenames().stream().filter(rename -> outputSourceFieldName.equals(rename.getSourceValueName())).findAny().orElse(null);
            // otherwise the derived field name is expected to be the same
            if (renameMapping != null) {
                derivedFieldName = renameMapping.getTargetValueName();
            } else if (renameFields && fieldRenames.containsKey(outputSourceFieldName)) {
                derivedFieldName = fieldRenames.get(outputSourceFieldName);
            }
            for (final Vertex outputTargetVertex : outputTargetVertices) {
                final String outputTargetName = outputTargetVertex.getProperty(DictionaryConst.PROPERTY_NAME);
                final Vertex derivedField = findFieldVertex(parentTransMeta, outputTargetName, derivedFieldName);
                // add an "inputs" link from the field of the input source step to the input target step
                metaverseBuilder.addLink(outputSourceField, DictionaryConst.LINK_INPUTS, outputTargetVertex);
                if (derivedField == null) {
                    LOGGER.debug(Messages.getString("WARN.TargetInputFieldNotFound", meta.getName(), outputSourceVertex.getProperty(DictionaryConst.PROPERTY_NAME), outputSourceField.getProperty(DictionaryConst.PROPERTY_NAME), subTransMeta.getName(), outputTargetName));
                } else {
                    // add a "derives" link from the field of the output source step to the output field of the output target step
                    metaverseBuilder.addLink(outputSourceField, DictionaryConst.LINK_DERIVES, derivedField);
                }
                // if the output target step has any virtual input fields that themselves aren't output by any step, remove
                // them; these are the orphaned output steps of the mapping step that the base analyzer code wasn't able to
                // resolve, and since the mapping step should not have any output fields, these can be removed
                final List<Vertex> allOutputTargetInputFields = IteratorUtils.toList(outputTargetVertex.getVertices(Direction.IN, DictionaryConst.LINK_INPUTS).iterator());
                for (final Vertex outputTargetInputField : allOutputTargetInputFields) {
                    if ("true".equals(outputTargetInputField.getProperty(DictionaryConst.NODE_VIRTUAL).toString())) {
                        // check further that this field does not have a containing step that outputs it
                        final Iterator<Vertex> parentSteps = outputTargetInputField.getVertices(Direction.IN, DictionaryConst.LINK_OUTPUTS).iterator();
                        boolean hasParent = false;
                        while (parentSteps.hasNext()) {
                            hasParent = true;
                            break;
                        }
                        if (!hasParent) {
                            outputTargetInputField.remove();
                        }
                    }
                }
            }
        }
    }
}

Example 7 with MappingIODefinition

use of org.pentaho.di.trans.steps.mapping.MappingIODefinition in project pentaho-kettle by pentaho.

the class SimpleMappingTest method setup.

@Before
public void setup() throws Exception {
    stepMockHelper = new StepMockHelper<SimpleMappingMeta, SimpleMappingData>("SIMPLE_MAPPING_TEST", SimpleMappingMeta.class, SimpleMappingData.class);
    when(stepMockHelper.logChannelInterfaceFactory.create(any(), any(LoggingObjectInterface.class))).thenReturn(stepMockHelper.logChannelInterface);
    when(stepMockHelper.trans.isRunning()).thenReturn(true);
    // Mock for MappingInput
    MappingInput mpInputMock = mock(MappingInput.class);
    when(mpInputMock.getStepname()).thenReturn(MAPPING_INPUT_STEP_NAME);
    // Mock for MappingOutput
    MappingOutput mpOutputMock = mock(MappingOutput.class);
    when(mpOutputMock.getStepname()).thenReturn(MAPPING_OUTPUT_STEP_NAME);
    // Mock for RowDataInputMapper
    RowDataInputMapper rdInputMpMock = mock(RowDataInputMapper.class);
    RowMetaInterface rwMetaInMock = mock(RowMeta.class);
    doReturn(Boolean.TRUE).when(rdInputMpMock).putRow(rwMetaInMock, new Object[] {});
    // Mock for RowProducer
    RowProducer rProducerMock = mock(RowProducer.class);
    when(rProducerMock.putRow(any(RowMetaInterface.class), any(Object[].class), anyBoolean())).thenReturn(true);
    // Mock for MappingIODefinition
    MappingIODefinition mpIODefMock = mock(MappingIODefinition.class);
    // Set up real SimpleMappingData with some mocked elements
    simpleMpData.mappingInput = mpInputMock;
    simpleMpData.mappingOutput = mpOutputMock;
    simpleMpData.rowDataInputMapper = rdInputMpMock;
    simpleMpData.mappingTrans = stepMockHelper.trans;
    when(stepMockHelper.trans.findStepInterface(MAPPING_OUTPUT_STEP_NAME, 0)).thenReturn(mpOutputMock);
    when(stepMockHelper.trans.addRowProducer(MAPPING_INPUT_STEP_NAME, 0)).thenReturn(rProducerMock);
    when(stepMockHelper.processRowsStepMetaInterface.getInputMapping()).thenReturn(mpIODefMock);
}

Example 8 with MappingIODefinition

use of org.pentaho.di.trans.steps.mapping.MappingIODefinition in project pentaho-kettle by pentaho.

the class SimpleMappingMeta method readRep.

public void readRep(Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases) throws KettleException {
    String method = rep.getStepAttributeString(id_step, "specification_method");
    specificationMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(method);
    String transId = rep.getStepAttributeString(id_step, "trans_object_id");
    transObjectId = Utils.isEmpty(transId) ? null : new StringObjectId(transId);
    transName = rep.getStepAttributeString(id_step, "trans_name");
    fileName = rep.getStepAttributeString(id_step, "filename");
    directoryPath = rep.getStepAttributeString(id_step, "directory_path");
    // Backward compatibility check for object specification
    // 
    checkObjectLocationSpecificationMethod();
    inputMapping = new MappingIODefinition(rep, id_step, "input_", 0);
    outputMapping = new MappingIODefinition(rep, id_step, "output_", 0);
    mappingParameters = new MappingParameters(rep, id_step);
}

Example 9 with MappingIODefinition

use of org.pentaho.di.trans.steps.mapping.MappingIODefinition in project pentaho-kettle by pentaho.

the class SimpleMappingMeta method loadXML.

public void loadXML(Node stepnode, List<DatabaseMeta> databases, IMetaStore metaStore) throws KettleXMLException {
    try {
        String method = XMLHandler.getTagValue(stepnode, "specification_method");
        specificationMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(method);
        String transId = XMLHandler.getTagValue(stepnode, "trans_object_id");
        transObjectId = Utils.isEmpty(transId) ? null : new StringObjectId(transId);
        transName = XMLHandler.getTagValue(stepnode, "trans_name");
        fileName = XMLHandler.getTagValue(stepnode, "filename");
        directoryPath = XMLHandler.getTagValue(stepnode, "directory_path");
        // Backward compatibility check for object specification
        // 
        checkObjectLocationSpecificationMethod();
        Node mappingsNode = XMLHandler.getSubNode(stepnode, "mappings");
        if (mappingsNode == null) {
            throw new KettleXMLException("Unable to find <mappings> element in the step XML");
        }
        // Read all the input mapping definitions...
        // 
        Node inputNode = XMLHandler.getSubNode(mappingsNode, "input");
        Node mappingNode = XMLHandler.getSubNode(inputNode, MappingIODefinition.XML_TAG);
        if (mappingNode != null) {
            inputMapping = new MappingIODefinition(mappingNode);
        } else {
            // empty
            inputMapping = new MappingIODefinition();
        }
        Node outputNode = XMLHandler.getSubNode(mappingsNode, "output");
        mappingNode = XMLHandler.getSubNode(outputNode, MappingIODefinition.XML_TAG);
        if (mappingNode != null) {
            outputMapping = new MappingIODefinition(mappingNode);
        } else {
            // empty
            outputMapping = new MappingIODefinition();
        }
        // Load the mapping parameters too..
        // 
        Node mappingParametersNode = XMLHandler.getSubNode(mappingsNode, MappingParameters.XML_TAG);
        mappingParameters = new MappingParameters(mappingParametersNode);
    } catch (Exception e) {
        throw new KettleXMLException(BaseMessages.getString(PKG, "SimpleMappingMeta.Exception.ErrorLoadingTransformationStepFromXML"), e);
    }
}