Examples with TransformationStepNode org.pentaho.metaverse.frames.TransformationStepNode used on opensource projects

Example 11 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testValueMapperStepNode_newField.

@Test
public void testValueMapperStepNode_newField() throws Exception {
    TransformationStepNode valueMapperStepNode = root.getStepNode("value_mapper", "Value Mapper - new field");
    assertEquals(1, getIterableSize(valueMapperStepNode.getStreamFieldNodesUses()));
    StreamFieldNode usesNode = valueMapperStepNode.getStreamFieldNodesUses().iterator().next();
    ValueMapperMeta meta = (ValueMapperMeta) getStepMeta(valueMapperStepNode);
    assertEquals(meta.getFieldToUse(), usesNode.getName());
    assertNull(usesNode.getOperations());
    Iterable<StreamFieldNode> inFields = valueMapperStepNode.getInputStreamFields();
    int countInputs = getIterableSize(inFields);
    Iterable<StreamFieldNode> outFields = valueMapperStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    assertEquals(countInputs + 1, countOutputs);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (outField.getName().equals(meta.getTargetField())) {
            Iterable<StreamFieldNode> fieldNodesThatDeriveMe = outField.getFieldNodesThatDeriveMe();
            assertNotNull(fieldNodesThatDeriveMe);
            int derivesCount = getIterableSize(outField.getFieldNodesThatDeriveMe());
            assertEquals(1, derivesCount);
            for (FieldNode derives : outField.getFieldNodesThatDeriveMe()) {
                assertEquals(usesNode.getName(), derives.getName());
                assertEquals(usesNode.getType(), derives.getType());
            }
            Operations ops = MetaverseUtil.convertOperationsStringToMap(outField.getOperations());
            List<IOperation> dataOps = ops.get(ChangeType.DATA);
            assertNotNull(dataOps);
            assertEquals(1, dataOps.size());
            for (IOperation dataOp : dataOps) {
                assertEquals(Operation.MAPPING_CATEGORY, dataOp.getCategory());
                assertEquals(DictionaryConst.PROPERTY_TRANSFORMS, dataOp.getName());
            }
            // there should not be any metadata operations
            assertNull(ops.get(ChangeType.METADATA));
        }
    }
}

Example 12 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testStringsReplaceStepNode.

@Test
public void testStringsReplaceStepNode() throws Exception {
    TransformationStepNode node = root.getStepNode("strings_replace", "Replace in string");
    // Make sure we have the right number of links used, created and derived. Also,
    // Ensure there is an entry in the operations for those fields that are derived.
    assertEquals(5, getIterableSize(node.getInputStreamFields()));
    assertEquals(5, getIterableSize(node.getStreamFieldNodesUses()));
    assertEquals(10, getIterableSize(node.getOutputStreamFields()));
    for (StreamFieldNode sfn : node.getOutputStreamFields()) {
        // The following are special cases for this test, they are passthrough fields
        if (sfn.getName().equals("LastName") || sfn.getName().equals("FirstName") || sfn.getName().equals("NickName") || sfn.getName().equals("Template")) {
            assertTrue(Const.isEmpty(sfn.getOperations()));
        } else {
            assertFalse(Const.isEmpty(sfn.getOperations()));
        }
    }
}

Example 13 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTransformationStepNodes.

@Test
public void testTransformationStepNodes() throws Exception {
    for (TransformationNode transNode : root.getTransformations()) {
        TransMeta tm = new TransMeta(new FileInputStream(transNode.getPath()), null, true, null, null);
        List<StepMeta> transMetaSteps = tm.getSteps();
        int matchCount = 0;
        for (StepMeta transMetaStep : transMetaSteps) {
            // let's see if the steps are in the graph for this transformation
            TransformationStepNode stepNode = transNode.getStepNode(transMetaStep.getName());
            assertNotNull(stepNode);
            assertEquals("Incorrect type", DictionaryConst.NODE_TYPE_TRANS_STEP, stepNode.getType());
            assertEquals("Incorrect entity type", DictionaryConst.NODE_TYPE_TRANS_STEP, stepNode.getEntity().getName());
            ++matchCount;
        }
        assertEquals("Not all transformation steps are modeled in the graph for [" + tm.getName() + "]", transMetaSteps.size(), matchCount);
        Collection<String> expectedStepNames = new TreeSet<>(transMetaSteps.stream().map(sm -> sm.getName()).collect(Collectors.toList()));
        Collection<String> actualStepNames = new TreeSet<>(StreamSupport.stream(transNode.getStepNodes().spliterator(), false).map(tsn -> tsn.asVertex().getProperty("name").toString()).collect(Collectors.toList()));
        assertEquals("Incorrect number of Steps in the graph for transformation [" + tm.getName() + "]", expectedStepNames, actualStepNames);
    }
}

Example 14 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testStringsCutStepNode.

@Test
public void testStringsCutStepNode() throws Exception {
    TransformationStepNode node = root.getStepNode("strings_cut", "Strings cut");
    // Make sure we have the right number of links used, created and derived. Also,
    // Ensure there is an entry in the operations for those fields that are derived.
    assertEquals(3, getIterableSize(node.getInputStreamFields()));
    assertEquals(3, getIterableSize(node.getStreamFieldNodesUses()));
    assertEquals(4, getIterableSize(node.getOutputStreamFields()));
    for (StreamFieldNode sfn : node.getOutputStreamFields()) {
        // "Last Name" is a special case for this test, it is passthrough
        if (sfn.getName().equals("Middle Name")) {
            assertTrue(Const.isEmpty(sfn.getOperations()));
        } else {
            assertFalse(Const.isEmpty(sfn.getOperations()));
        }
    }
}

Example 15 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MdiValidationIT method verifyInjectorTextFileOutputNode.

private void verifyInjectorTextFileOutputNode(final TransformationStepNode mdiNode, final String outputFileName, final String[] expectedOutputFieldNameArray, final TransformationStepNode templateReadFromStepNode) {
    final List<TransformationStepNode> mdiHopsToNodes = IteratorUtils.toList(mdiNode.getNextSteps().iterator());
    assertEquals(1, mdiHopsToNodes.size());
    // verify that the text file output step writes to the correct output file
    final TransformationStepNode textFileOutputNode = mdiHopsToNodes.get(0);
    assertEquals("Text file output", textFileOutputNode.getName());
    final List<FileNode> writesToNodes = IteratorUtils.toList(textFileOutputNode.getWritesToFileNodes().iterator());
    assertEquals(1, writesToNodes.size());
    final FileNode fileNode = writesToNodes.get(0);
    assertTrue(String.format("Path expected to end with '%s': %s", outputFileName, fileNode.getPath()), fileNode.getPath().endsWith(outputFileName));
    // verify that the mdi node output fields are inputs into the text file output step
    final List<StreamFieldNode> mdiOutputFields = IteratorUtils.toList(mdiNode.getOutputStreamFields().iterator());
    final List<StreamFieldNode> mdiInputFields = IteratorUtils.toList(mdiNode.getInputStreamFields().iterator());
    final List<StreamFieldNode> inputFields = IteratorUtils.toList(textFileOutputNode.getInputStreamFields().iterator());
    assertEquals(mdiOutputFields.size(), inputFields.size());
    final List<StreamFieldNode> tempalteReadFromStepOutputFields = templateReadFromStepNode == null ? new ArrayList() : IteratorUtils.toList(templateReadFromStepNode.getOutputStreamFields().iterator());
    for (final StreamFieldNode field : inputFields) {
        assertTrue(mdiOutputFields.contains(field));
        // template node that is being read and that the same field is also an input into the mdi step node
        if (templateReadFromStepNode != null) {
            // there should be 1 field that derives this field, and it should be one of the mdi input fields, as well as
            // one of the output fields from the template step that is being read from
            assertEquals(1, getIterableSize(field.getFieldNodesThatDeriveMe()));
            final StreamFieldNode derivingField = (StreamFieldNode) IteratorUtils.toList(field.getFieldNodesThatDeriveMe().iterator()).get(0);
            assertTrue(mdiInputFields.contains(derivingField));
            assertTrue(tempalteReadFromStepOutputFields.contains(derivingField));
            assertEquals(field.getName(), derivingField.getName());
        }
    }
    // verify that the text file output step has the expected output fields, and that each field is derived from an
    // mdi output field with the same name
    final List<StreamFieldNode> outputFields = IteratorUtils.toList(textFileOutputNode.getOutputStreamFields().iterator());
    final List<String> expectedOutputFieldNames = Arrays.asList(expectedOutputFieldNameArray == null ? new String[] {} : expectedOutputFieldNameArray);
    assertEquals(expectedOutputFieldNames.size(), outputFields.size());
    for (final StreamFieldNode field : outputFields) {
        assertTrue(expectedOutputFieldNames.contains(field.getName()));
        // there should be 1 field that derives this field, and it should be one of the mdi output fields
        assertEquals(1, getIterableSize(field.getFieldNodesThatDeriveMe()));
        final StreamFieldNode derivingField = (StreamFieldNode) IteratorUtils.toList(field.getFieldNodesThatDeriveMe().iterator()).get(0);
        assertTrue(mdiOutputFields.contains(derivingField));
        assertEquals(field.getName(), derivingField.getName());
    }
}