Examples with TransformationStepNode org.pentaho.metaverse.frames.TransformationStepNode used on opensource projects

Example 1 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MappingAnalyzerValidationIT method test_oneIONonMainPathWithMappingsAndExtraField.

@Test
public void test_oneIONonMainPathWithMappingsAndExtraField() throws Exception {
    final String transNodeName = "oneIONonMainPathWithMappingsAndExtraField";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode("sub", true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 35, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 102, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { RANDOM_INT, RANDOM_INT, RANDOM_INT, RANDOM_VALUE, RANDOM_VALUE, RANDOM_VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, CHECKSUM, CHECKSUM, CHECKSUM, CHECKSUM, NEW_CHECKSUM })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Data grid", "Write to log - Data grid", "mapping - calc checksum", "Write to log - mapping - calc checksum", "Write to log Checksum" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input checksum", "calc checksum", "output checksum", "Write to log - Input checksum", "Write to log - calc checksum", "Write to log - output checksum" }, false);
    final TransformationStepNode dataGrid = (TransformationStepNode) parentStepNodeMap.get("Data grid");
    final TransformationStepNode writeToLogDataGrid = (TransformationStepNode) parentStepNodeMap.get("Write to log - Data grid");
    final TransformationStepNode calcChecksum = (TransformationStepNode) parentStepNodeMap.get("mapping - calc checksum");
    final TransformationStepNode writeToLogCalcChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log - mapping - calc checksum");
    final TransformationStepNode writeToLogChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log Checksum");
    // sub-trans nodes within the parent graph
    final TransformationStepNode inputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Input checksum");
    final TransformationStepNode calcChecksumSubTrans = (TransformationStepNode) subTransStepNodeMap.get("calc checksum");
    final TransformationStepNode outputChecksum = (TransformationStepNode) subTransStepNodeMap.get("output checksum");
    final TransformationStepNode writeToLogOutputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Write to log - output checksum");
    // ---------- Data grid
    verifyNodes(IteratorUtils.toList(dataGrid.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(dataGrid.getNextSteps().iterator()), testLineageNode(calcChecksum), testLineageNode(writeToLogDataGrid));
    verifyNodes(IteratorUtils.toList(dataGrid.getOutputStreamFields().iterator()), testFieldNode(RANDOM_VALUE), testFieldNode(RANDOM_INT));
    assertEquals(2, getIterableSize(dataGrid.getAllInNodes()));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(4, getIterableSize(dataGrid.getAllOutNodes()));
    // ---------- mapping - calc checksum
    verifyNodes(IteratorUtils.toList(calcChecksum.getPreviousSteps().iterator()), testLineageNode(dataGrid));
    verifyNodes(IteratorUtils.toList(calcChecksum.getNextSteps().iterator()), testLineageNode(writeToLogChecksum), testLineageNode(writeToLogCalcChecksum));
    verifyNodes(IteratorUtils.toList(calcChecksum.getInputStreamFields().iterator()), testFieldNode(RANDOM_VALUE), testFieldNode(RANDOM_INT));
    verifyNodes(IteratorUtils.toList(calcChecksum.getOutputStreamFields().iterator()));
    assertEquals(5, getIterableSize(calcChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(calcChecksum.getAllOutNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getOutNodes(LINK_EXECUTES)));
    // ---------- Write to log - mapping - calc checksum
    verifyNodes(IteratorUtils.toList(writeToLogCalcChecksum.getOutputStreamFields().iterator()));
    // verify properties
    verifyNodeProperties(calcChecksum, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] description,input [1] update field names,input [1] rename " + "[1],output [1],output [1] description,output [1] update field names,output [1] rename [1]").put("input [1]", "Data grid > [sub] Input checksum").put("input [1] update field names", "true").put("input [1] rename [1]", "randomValue > value").put("output [1]", "[sub] output checksum > Write to log Checksum").put("output [1] rename [1]", "checksum > newChecksum").put("output [1] update field names", "false").build());
    // ---------- Write to log Checksum
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksum.getName()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getNextSteps().iterator()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getOutputStreamFields().iterator()), testFieldNode(NEW_CHECKSUM), testFieldNode(RANDOM_VALUE), testFieldNode(RANDOM_INT));
    verifyStepIOLinks(writeToLogChecksum, testLineageLink(testFieldNode(CHECKSUM), LINK_DERIVES, testFieldNode(NEW_CHECKSUM)), testLineageLink(testFieldNode(VALUE), LINK_DERIVES, testFieldNode(RANDOM_VALUE)));
    assertEquals(5, getIterableSize(writeToLogChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(writeToLogChecksum.getAllOutNodes()));
    // ---------- output checksum
    verifyNodes(IteratorUtils.toList(outputChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksumSubTrans.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getNextSteps().iterator()), testStepNode(writeToLogOutputChecksum.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(outputChecksum.getOutputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    assertEquals(5, getIterableSize(outputChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(outputChecksum.getAllOutNodes()));
    // Verify the following link chains
    // - chain 1: Data grid > outputs > randomValue >  inputs > Input checksum > outputs > value > inputs >
    // calc checksum > outputs > value > inputs > calc checksum > outputs > value > inputs > Write to log Checksum >
    // outputs > randomValue
    // - chain 2: Data grid  > outputs > randomValue > derives > Input Checksum:value > derives
    // > calc checksum:value > derives > output checksum:value > derives > Write to log Checksum:randomValue
    final FramedMetaverseNode dataGrid_output_randomValue = verifyLinkedNode(dataGrid, LINK_OUTPUTS, RANDOM_VALUE);
    final FramedMetaverseNode inputChecksum_output_value = verifyLinkedNode(inputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode calcChecksum_output_value = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode outputChecksum_output_value = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode writeToLogChecksum_output_randomValue = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, RANDOM_VALUE);
    assertEquals(inputChecksum, verifyLinkedNode(dataGrid_output_randomValue, LINK_INPUTS, inputChecksum.getName()));
    assertEquals(calcChecksumSubTrans, verifyLinkedNode(inputChecksum_output_value, LINK_INPUTS, calcChecksumSubTrans.getName()));
    assertEquals(outputChecksum, verifyLinkedNode(calcChecksum_output_value, LINK_INPUTS, outputChecksum.getName()));
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_value, LINK_INPUTS, writeToLogChecksum.getName()));
    assertTrue(IteratorUtils.toList(dataGrid_output_randomValue.getOutNodes(LINK_DERIVES).iterator()).contains(inputChecksum_output_value));
    assertTrue(IteratorUtils.toList(inputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(calcChecksum_output_value));
    assertTrue(IteratorUtils.toList(calcChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(outputChecksum_output_value));
    assertTrue(IteratorUtils.toList(outputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(writeToLogChecksum_output_randomValue));
    // Verify the following link chains
    // - chain 1: calc checksum > outputs > checksum > inputs > output checksum > output > checksum > inputs > Write to
    // log Checksum > outputs > newChecksum
    // - chain 2: calc checksum:checksum > derives > output checksum:checksum > derives > Write to log
    // Checksum:newChecksum
    final FramedMetaverseNode calcChecksum_output_checksum = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode outputChecksum_output_checksum = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode writeToLogChecksum_output_newChecksum = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, NEW_CHECKSUM);
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_checksum, LINK_INPUTS, writeToLogChecksum.getName()));
    assertEquals(writeToLogChecksum_output_newChecksum, verifyLinkedNode(outputChecksum_output_checksum, LINK_DERIVES, NEW_CHECKSUM));
}

Example 2 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MappingAnalyzerValidationIT method test_oneIONonMainPathNoFieldMappings.

@Test
public void test_oneIONonMainPathNoFieldMappings() throws Exception {
    final String transNodeName = "oneIONonMainPathNoFieldMappings";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode("sub", true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 32, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 92, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, CHECKSUM, CHECKSUM, CHECKSUM, CHECKSUM, CHECKSUM })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Data grid", "mapping - calc checksum", "Write to log Checksum", "Write to log - Data grid", "Write to log - mapping - calc checksum" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input checksum", "calc checksum", "output checksum", "Write to log - Input checksum", "Write to log - calc checksum", "Write to log - output checksum" }, false);
    final TransformationStepNode dataGrid = (TransformationStepNode) parentStepNodeMap.get("Data grid");
    final TransformationStepNode writeToLogDataGrid = (TransformationStepNode) parentStepNodeMap.get("Write to log - Data grid");
    final TransformationStepNode calcChecksum = (TransformationStepNode) parentStepNodeMap.get("mapping - calc checksum");
    final TransformationStepNode writeToLogCalcChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log - mapping - calc checksum");
    final TransformationStepNode writeToLogChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log Checksum");
    // sub-trans nodes within the parent graph
    final TransformationStepNode inputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Input checksum");
    final TransformationStepNode calcChecksumSubTrans = (TransformationStepNode) subTransStepNodeMap.get("calc checksum");
    final TransformationStepNode outputChecksum = (TransformationStepNode) subTransStepNodeMap.get("output checksum");
    final TransformationStepNode writeToLogOutputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Write to log - output checksum");
    // ---------- Data grid
    verifyNodes(IteratorUtils.toList(dataGrid.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(dataGrid.getNextSteps().iterator()), testLineageNode(calcChecksum), testLineageNode(writeToLogDataGrid));
    verifyNodes(IteratorUtils.toList(dataGrid.getOutputStreamFields().iterator()), testFieldNode(VALUE));
    assertEquals(2, getIterableSize(dataGrid.getAllInNodes()));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(dataGrid.getAllOutNodes()));
    // ---------- mapping - calc checksum
    verifyNodes(IteratorUtils.toList(calcChecksum.getPreviousSteps().iterator()), testLineageNode(dataGrid));
    verifyNodes(IteratorUtils.toList(calcChecksum.getNextSteps().iterator()), testLineageNode(writeToLogChecksum), testLineageNode(writeToLogCalcChecksum));
    verifyNodes(IteratorUtils.toList(calcChecksum.getInputStreamFields().iterator()), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(calcChecksum.getOutputStreamFields().iterator()));
    assertEquals(4, getIterableSize(calcChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(calcChecksum.getAllOutNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getOutNodes(LINK_EXECUTES)));
    // ---------- Write to log - mapping - calc checksum
    verifyNodes(IteratorUtils.toList(writeToLogCalcChecksum.getOutputStreamFields().iterator()));
    // verify properties
    verifyNodeProperties(calcChecksum, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] description,input [1] update field names,output [1],output" + " [1] description,output [1] update field names").put("input [1]", "Data grid > [sub] Input checksum").put("input [1] update field names", "false").put("output [1]", "[sub] output checksum > Write to log Checksum").put("output [1] update field names", "false").build());
    // ---------- Write to log Checksum
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksum.getName()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getNextSteps().iterator()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getOutputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyStepIOLinks(writeToLogChecksum, testLineageLink(testFieldNode(CHECKSUM), LINK_DERIVES, testFieldNode(CHECKSUM)), testLineageLink(testFieldNode(VALUE), LINK_DERIVES, testFieldNode(VALUE)));
    assertEquals(5, getIterableSize(writeToLogChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(writeToLogChecksum.getAllOutNodes()));
    // ---------- output checksum
    verifyNodes(IteratorUtils.toList(outputChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksumSubTrans.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getNextSteps().iterator()), testStepNode(writeToLogOutputChecksum.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(outputChecksum.getOutputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    assertEquals(5, getIterableSize(outputChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(outputChecksum.getAllOutNodes()));
    // Verify the following link chains
    // - chain 1: Data grid > outputs > value >  inputs > Input checksum > outputs > value > inputs >
    // calc checksum > outputs > value > input > output checksum > outputs > checksum > outputs > value > inputs >
    // Write to log Checksum > outputs > value
    // - chain 2: Data grid > outputs > value > derives > Input Checksum:value > derives
    // > calc checksum:value > derives > output checksum:value > derives > Write to log Checksum:value
    final FramedMetaverseNode dataGrid_output_value = verifyLinkedNode(dataGrid, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode inputChecksum_output_value = verifyLinkedNode(inputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode calcChecksum_output_value = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode outputChecksum_output_value = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode writeToLogChecksum_output_value = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, VALUE);
    assertEquals(inputChecksum, verifyLinkedNode(dataGrid_output_value, LINK_INPUTS, inputChecksum.getName()));
    assertEquals(calcChecksumSubTrans, verifyLinkedNode(inputChecksum_output_value, LINK_INPUTS, calcChecksumSubTrans.getName()));
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_value, LINK_INPUTS, writeToLogChecksum.getName()));
    assertTrue(IteratorUtils.toList(dataGrid_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(inputChecksum_output_value));
    assertTrue(IteratorUtils.toList(inputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(calcChecksum_output_value));
    assertTrue(IteratorUtils.toList(calcChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(outputChecksum_output_value));
    assertTrue(IteratorUtils.toList(outputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(writeToLogChecksum_output_value));
    // Verify the following link chains
    // - chain 1: calc checksum > outputs > checksum > inputs > output > output checksum > inputs > Write to log
    // Checksum > outputs > checksum
    // - chain 2: calc checksum:checksum > derives > output checksum:checksum > derives > Write to log
    // Checksum:checksum
    final FramedMetaverseNode calcChecksum_output_checksum = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode outputChecksum_output_checksum = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode writeToLogChecksum_output_checksum = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, CHECKSUM);
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_checksum, LINK_INPUTS, writeToLogChecksum.getName()));
    assertTrue(IteratorUtils.toList(outputChecksum_output_checksum.getOutNodes(LINK_DERIVES).iterator()).contains(writeToLogChecksum_output_checksum));
}

Example 3 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class SimpleMappingAnalyzerValidationIT method testMoreThanOneIOstepsMappingOut.

@Test
public void testMoreThanOneIOstepsMappingOut() throws Exception {
    final String transNodeName = "simple_moreThanOneIOstepsMappingOut";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode("simple_sub", true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 21, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 53, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "simple_sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { RANDOM_VALUE, RANDOM_VALUE, VALUE, VALUE, VALUE, PARITY, PARITY, NEW_PARITY })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Generate random integer", "Simple mapping (sub-transformation)", "Write to log Parity" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input parity", "output parity", "calc parity" }, false);
    final TransformationStepNode generateRandomInt = (TransformationStepNode) parentStepNodeMap.get("Generate random integer");
    final TransformationStepNode simpleMapping = (TransformationStepNode) parentStepNodeMap.get("Simple mapping (sub-transformation)");
    final TransformationStepNode writeToLogParity = (TransformationStepNode) parentStepNodeMap.get("Write to log Parity");
    // virtual sub-trans nodes within the parent graph
    final TransformationStepNode inputParity = (TransformationStepNode) subTransStepNodeMap.get("Input parity");
    final TransformationStepNode calcParity = (TransformationStepNode) subTransStepNodeMap.get("calc parity");
    final TransformationStepNode outputParity = (TransformationStepNode) subTransStepNodeMap.get("output parity");
    // ---------- Generate Random Int
    verifyNodes(IteratorUtils.toList(generateRandomInt.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(generateRandomInt.getNextSteps().iterator()), testLineageNode(simpleMapping));
    verifyNodes(IteratorUtils.toList(generateRandomInt.getOutputStreamFields().iterator()), testFieldNode(RANDOM_VALUE, false));
    assertEquals(2, getIterableSize(generateRandomInt.getAllInNodes()));
    assertEquals(1, getIterableSize(generateRandomInt.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(generateRandomInt.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(generateRandomInt.getAllOutNodes()));
    // ---------- "Simple mapping (sub-transformation)
    verifyNodes(IteratorUtils.toList(simpleMapping.getPreviousSteps().iterator()), testLineageNode(generateRandomInt));
    verifyNodes(IteratorUtils.toList(simpleMapping.getInputStreamFields().iterator()), testFieldNode(RANDOM_VALUE, false));
    verifyNodes(IteratorUtils.toList(simpleMapping.getOutputStreamFields().iterator()));
    assertEquals(4, getIterableSize(simpleMapping.getAllInNodes()));
    assertEquals(1, getIterableSize(simpleMapping.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(simpleMapping.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(simpleMapping.getAllOutNodes()));
    assertEquals(1, getIterableSize(simpleMapping.getOutNodes(LINK_EXECUTES)));
    // verify properties
    verifyNodeProperties(simpleMapping, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] update field names,input [1] rename [1],output [1],output [1] " + "update field names,output [1] rename [1]").put("input [1]", "Generate random integer > [simple_sub] Input parity").put("input [1] update field names", "true").put("input [1] rename [1]", "randomValue > value").put("output [1]", "[simple_sub] output parity > Write to log Parity").put("output [1] rename [1]", "parity > newParity").put("output [1] update field names", "false").build());
    // ---------- output parity
    verifyNodes(IteratorUtils.toList(outputParity.getPreviousSteps().iterator()), testStepNode(calcParity.getName(), false));
    verifyNodes(IteratorUtils.toList(outputParity.getNextSteps().iterator()));
    verifyNodes(IteratorUtils.toList(outputParity.getInputStreamFields().iterator()), testFieldNode(PARITY, false), testFieldNode(VALUE, false));
    verifyNodes(IteratorUtils.toList(outputParity.getOutputStreamFields().iterator()), testFieldNode(PARITY, false), testFieldNode(VALUE, false));
    assertEquals(5, getIterableSize(outputParity.getAllInNodes()));
    assertEquals(1, getIterableSize(outputParity.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(outputParity.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(outputParity.getAllOutNodes()));
    // Verify the following link chains
    // - chain 1: Generate Random Int > outputs > randomValue >  inputs > Input parity > outputs > value > inputs >
    // calc parity > outputs > value > inputs > output parity > outputs > value > inputs
    // > Write to log Parity > outputs > randomValue
    // - chain 2: Generate Random Int > outputs > randomValue > derives > Input Parity:value > derives
    // > calc parity:value > derives > output parity:value > derives > Write to log Parity:randomValue
    final FramedMetaverseNode generateRandomInt_output_randomValue = verifyLinkedNode(generateRandomInt, LINK_OUTPUTS, RANDOM_VALUE);
    final FramedMetaverseNode inputParity_output_value = verifyLinkedNode(inputParity, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode calcParity_output_value = verifyLinkedNode(calcParity, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode outputParity_output_value = verifyLinkedNode(outputParity, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode writeToLogParity_output_randomValue = verifyLinkedNode(writeToLogParity, LINK_OUTPUTS, RANDOM_VALUE);
    assertEquals(inputParity, verifyLinkedNode(generateRandomInt_output_randomValue, LINK_INPUTS, inputParity.getName()));
    assertEquals(calcParity, verifyLinkedNode(inputParity_output_value, LINK_INPUTS, calcParity.getName()));
    assertEquals(outputParity, verifyLinkedNode(calcParity_output_value, LINK_INPUTS, outputParity.getName()));
    assertEquals(writeToLogParity, verifyLinkedNode(outputParity_output_value, LINK_INPUTS, writeToLogParity.getName()));
    assertEquals(inputParity_output_value, verifyLinkedNode(generateRandomInt_output_randomValue, LINK_DERIVES, VALUE));
    assertEquals(calcParity_output_value, verifyLinkedNode(inputParity_output_value, LINK_DERIVES, VALUE));
    assertEquals(outputParity_output_value, verifyLinkedNode(calcParity_output_value, LINK_DERIVES, VALUE));
    assertEquals(writeToLogParity_output_randomValue, verifyLinkedNode(outputParity_output_value, LINK_DERIVES, RANDOM_VALUE));
    // Verify the following link chains
    // - chain 1: calc parity > outputs > parity > inputs > output parity > outputs > parity > inputs
    // > Write to log Parity > outputs > newParity
    // - chain 2: calc parity:parity > derives > output parity:parity > derives > Write to log
    // Parity:newParity
    final FramedMetaverseNode calcParity_output_parity = verifyLinkedNode(calcParity, LINK_OUTPUTS, PARITY);
    final FramedMetaverseNode outputParity_output_parity = verifyLinkedNode(outputParity, LINK_OUTPUTS, PARITY);
    final FramedMetaverseNode writeToLogParity_output_newParity = verifyLinkedNode(writeToLogParity, LINK_OUTPUTS, NEW_PARITY);
    assertEquals(outputParity, verifyLinkedNode(calcParity_output_parity, LINK_INPUTS, outputParity.getName()));
    assertEquals(writeToLogParity, verifyLinkedNode(outputParity_output_parity, LINK_INPUTS, writeToLogParity.getName()));
    assertEquals(outputParity_output_parity, verifyLinkedNode(calcParity_output_parity, LINK_DERIVES, PARITY));
    assertEquals(writeToLogParity_output_newParity, verifyLinkedNode(outputParity_output_parity, LINK_DERIVES, NEW_PARITY));
}

Example 4 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testNumberRangeStepNode_newField.

@Test
public void testNumberRangeStepNode_newField() throws Exception {
    TransformationStepNode numberRangeStepNode = root.getStepNode("number_range", "Number range");
    assertEquals(1, getIterableSize(numberRangeStepNode.getStreamFieldNodesUses()));
    StreamFieldNode usesNode = null;
    for (StreamFieldNode node : numberRangeStepNode.getStreamFieldNodesUses()) {
        usesNode = node;
        break;
    }
    NumberRangeMeta meta = (NumberRangeMeta) getStepMeta(numberRangeStepNode);
    assertEquals(meta.getInputField(), usesNode.getName());
    assertNull(usesNode.getOperations());
    Iterable<StreamFieldNode> outFields = numberRangeStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    assertEquals(2, countOutputs);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (outField.getName().equals(meta.getOutputField())) {
            Iterable<StreamFieldNode> fieldNodesThatDeriveMe = outField.getFieldNodesThatDeriveMe();
            assertNotNull(fieldNodesThatDeriveMe);
            int derivesCount = getIterableSize(outField.getFieldNodesThatDeriveMe());
            assertEquals(1, derivesCount);
            for (FieldNode derives : outField.getFieldNodesThatDeriveMe()) {
                assertEquals(usesNode.getName(), derives.getName());
                assertEquals(usesNode.getType(), derives.getType());
            }
            Operations ops = MetaverseUtil.convertOperationsStringToMap(outField.getOperations());
            List<IOperation> dataOps = ops.get(ChangeType.DATA);
            assertNotNull(dataOps);
            assertEquals(3, dataOps.size());
            for (IOperation dataOp : dataOps) {
                assertEquals(Operation.MAPPING_CATEGORY, dataOp.getCategory());
                assertEquals(DictionaryConst.PROPERTY_TRANSFORMS, dataOp.getName());
                assertTrue(dataOp.toString().contains(meta.getInputField()));
            }
            // there should not be any metadata operations
            assertNull(ops.get(ChangeType.METADATA));
        }
    }
}

Example 5 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testStringOperationsStepNode.

@Test
public void testStringOperationsStepNode() throws Exception {
    TransformationStepNode node = root.getStepNode("string_operations", "String operations");
    // Make sure we have the right number of links used
    assertEquals(4, getIterableSize(node.getInputStreamFields()));
    assertEquals(4, getIterableSize(node.getStreamFieldNodesUses()));
    assertEquals(5, getIterableSize(node.getOutputStreamFields()));
    for (StreamFieldNode sfn : node.getOutputStreamFields()) {
        // "Last Name" is a special case for this test, it is passthrough
        if (sfn.getName().equals("Last Name")) {
            assertTrue(Const.isEmpty(sfn.getOperations()));
        } else {
            assertFalse(Const.isEmpty(sfn.getOperations()));
        }
    }
}