Examples with TransformationNode org.pentaho.metaverse.frames.TransformationNode used on opensource projects

Example 1 with TransformationNode

use of org.pentaho.metaverse.frames.TransformationNode in project pentaho-metaverse by pentaho.

the class MappingAnalyzerValidationIT method test_oneIONonMainPathWithMappingsAndExtraField.

@Test
public void test_oneIONonMainPathWithMappingsAndExtraField() throws Exception {
    final String transNodeName = "oneIONonMainPathWithMappingsAndExtraField";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode("sub", true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 35, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 102, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { RANDOM_INT, RANDOM_INT, RANDOM_INT, RANDOM_VALUE, RANDOM_VALUE, RANDOM_VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, CHECKSUM, CHECKSUM, CHECKSUM, CHECKSUM, NEW_CHECKSUM })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Data grid", "Write to log - Data grid", "mapping - calc checksum", "Write to log - mapping - calc checksum", "Write to log Checksum" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input checksum", "calc checksum", "output checksum", "Write to log - Input checksum", "Write to log - calc checksum", "Write to log - output checksum" }, false);
    final TransformationStepNode dataGrid = (TransformationStepNode) parentStepNodeMap.get("Data grid");
    final TransformationStepNode writeToLogDataGrid = (TransformationStepNode) parentStepNodeMap.get("Write to log - Data grid");
    final TransformationStepNode calcChecksum = (TransformationStepNode) parentStepNodeMap.get("mapping - calc checksum");
    final TransformationStepNode writeToLogCalcChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log - mapping - calc checksum");
    final TransformationStepNode writeToLogChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log Checksum");
    // sub-trans nodes within the parent graph
    final TransformationStepNode inputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Input checksum");
    final TransformationStepNode calcChecksumSubTrans = (TransformationStepNode) subTransStepNodeMap.get("calc checksum");
    final TransformationStepNode outputChecksum = (TransformationStepNode) subTransStepNodeMap.get("output checksum");
    final TransformationStepNode writeToLogOutputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Write to log - output checksum");
    // ---------- Data grid
    verifyNodes(IteratorUtils.toList(dataGrid.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(dataGrid.getNextSteps().iterator()), testLineageNode(calcChecksum), testLineageNode(writeToLogDataGrid));
    verifyNodes(IteratorUtils.toList(dataGrid.getOutputStreamFields().iterator()), testFieldNode(RANDOM_VALUE), testFieldNode(RANDOM_INT));
    assertEquals(2, getIterableSize(dataGrid.getAllInNodes()));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(4, getIterableSize(dataGrid.getAllOutNodes()));
    // ---------- mapping - calc checksum
    verifyNodes(IteratorUtils.toList(calcChecksum.getPreviousSteps().iterator()), testLineageNode(dataGrid));
    verifyNodes(IteratorUtils.toList(calcChecksum.getNextSteps().iterator()), testLineageNode(writeToLogChecksum), testLineageNode(writeToLogCalcChecksum));
    verifyNodes(IteratorUtils.toList(calcChecksum.getInputStreamFields().iterator()), testFieldNode(RANDOM_VALUE), testFieldNode(RANDOM_INT));
    verifyNodes(IteratorUtils.toList(calcChecksum.getOutputStreamFields().iterator()));
    assertEquals(5, getIterableSize(calcChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(calcChecksum.getAllOutNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getOutNodes(LINK_EXECUTES)));
    // ---------- Write to log - mapping - calc checksum
    verifyNodes(IteratorUtils.toList(writeToLogCalcChecksum.getOutputStreamFields().iterator()));
    // verify properties
    verifyNodeProperties(calcChecksum, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] description,input [1] update field names,input [1] rename " + "[1],output [1],output [1] description,output [1] update field names,output [1] rename [1]").put("input [1]", "Data grid > [sub] Input checksum").put("input [1] update field names", "true").put("input [1] rename [1]", "randomValue > value").put("output [1]", "[sub] output checksum > Write to log Checksum").put("output [1] rename [1]", "checksum > newChecksum").put("output [1] update field names", "false").build());
    // ---------- Write to log Checksum
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksum.getName()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getNextSteps().iterator()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getOutputStreamFields().iterator()), testFieldNode(NEW_CHECKSUM), testFieldNode(RANDOM_VALUE), testFieldNode(RANDOM_INT));
    verifyStepIOLinks(writeToLogChecksum, testLineageLink(testFieldNode(CHECKSUM), LINK_DERIVES, testFieldNode(NEW_CHECKSUM)), testLineageLink(testFieldNode(VALUE), LINK_DERIVES, testFieldNode(RANDOM_VALUE)));
    assertEquals(5, getIterableSize(writeToLogChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(writeToLogChecksum.getAllOutNodes()));
    // ---------- output checksum
    verifyNodes(IteratorUtils.toList(outputChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksumSubTrans.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getNextSteps().iterator()), testStepNode(writeToLogOutputChecksum.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(outputChecksum.getOutputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    assertEquals(5, getIterableSize(outputChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(outputChecksum.getAllOutNodes()));
    // Verify the following link chains
    // - chain 1: Data grid > outputs > randomValue >  inputs > Input checksum > outputs > value > inputs >
    // calc checksum > outputs > value > inputs > calc checksum > outputs > value > inputs > Write to log Checksum >
    // outputs > randomValue
    // - chain 2: Data grid  > outputs > randomValue > derives > Input Checksum:value > derives
    // > calc checksum:value > derives > output checksum:value > derives > Write to log Checksum:randomValue
    final FramedMetaverseNode dataGrid_output_randomValue = verifyLinkedNode(dataGrid, LINK_OUTPUTS, RANDOM_VALUE);
    final FramedMetaverseNode inputChecksum_output_value = verifyLinkedNode(inputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode calcChecksum_output_value = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode outputChecksum_output_value = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode writeToLogChecksum_output_randomValue = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, RANDOM_VALUE);
    assertEquals(inputChecksum, verifyLinkedNode(dataGrid_output_randomValue, LINK_INPUTS, inputChecksum.getName()));
    assertEquals(calcChecksumSubTrans, verifyLinkedNode(inputChecksum_output_value, LINK_INPUTS, calcChecksumSubTrans.getName()));
    assertEquals(outputChecksum, verifyLinkedNode(calcChecksum_output_value, LINK_INPUTS, outputChecksum.getName()));
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_value, LINK_INPUTS, writeToLogChecksum.getName()));
    assertTrue(IteratorUtils.toList(dataGrid_output_randomValue.getOutNodes(LINK_DERIVES).iterator()).contains(inputChecksum_output_value));
    assertTrue(IteratorUtils.toList(inputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(calcChecksum_output_value));
    assertTrue(IteratorUtils.toList(calcChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(outputChecksum_output_value));
    assertTrue(IteratorUtils.toList(outputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(writeToLogChecksum_output_randomValue));
    // Verify the following link chains
    // - chain 1: calc checksum > outputs > checksum > inputs > output checksum > output > checksum > inputs > Write to
    // log Checksum > outputs > newChecksum
    // - chain 2: calc checksum:checksum > derives > output checksum:checksum > derives > Write to log
    // Checksum:newChecksum
    final FramedMetaverseNode calcChecksum_output_checksum = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode outputChecksum_output_checksum = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode writeToLogChecksum_output_newChecksum = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, NEW_CHECKSUM);
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_checksum, LINK_INPUTS, writeToLogChecksum.getName()));
    assertEquals(writeToLogChecksum_output_newChecksum, verifyLinkedNode(outputChecksum_output_checksum, LINK_DERIVES, NEW_CHECKSUM));
}

Example 2 with TransformationNode

use of org.pentaho.metaverse.frames.TransformationNode in project pentaho-metaverse by pentaho.

the class MappingAnalyzerValidationIT method test_oneIONonMainPathNoFieldMappings.

@Test
public void test_oneIONonMainPathNoFieldMappings() throws Exception {
    final String transNodeName = "oneIONonMainPathNoFieldMappings";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode("sub", true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 32, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 92, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, CHECKSUM, CHECKSUM, CHECKSUM, CHECKSUM, CHECKSUM })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Data grid", "mapping - calc checksum", "Write to log Checksum", "Write to log - Data grid", "Write to log - mapping - calc checksum" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input checksum", "calc checksum", "output checksum", "Write to log - Input checksum", "Write to log - calc checksum", "Write to log - output checksum" }, false);
    final TransformationStepNode dataGrid = (TransformationStepNode) parentStepNodeMap.get("Data grid");
    final TransformationStepNode writeToLogDataGrid = (TransformationStepNode) parentStepNodeMap.get("Write to log - Data grid");
    final TransformationStepNode calcChecksum = (TransformationStepNode) parentStepNodeMap.get("mapping - calc checksum");
    final TransformationStepNode writeToLogCalcChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log - mapping - calc checksum");
    final TransformationStepNode writeToLogChecksum = (TransformationStepNode) parentStepNodeMap.get("Write to log Checksum");
    // sub-trans nodes within the parent graph
    final TransformationStepNode inputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Input checksum");
    final TransformationStepNode calcChecksumSubTrans = (TransformationStepNode) subTransStepNodeMap.get("calc checksum");
    final TransformationStepNode outputChecksum = (TransformationStepNode) subTransStepNodeMap.get("output checksum");
    final TransformationStepNode writeToLogOutputChecksum = (TransformationStepNode) subTransStepNodeMap.get("Write to log - output checksum");
    // ---------- Data grid
    verifyNodes(IteratorUtils.toList(dataGrid.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(dataGrid.getNextSteps().iterator()), testLineageNode(calcChecksum), testLineageNode(writeToLogDataGrid));
    verifyNodes(IteratorUtils.toList(dataGrid.getOutputStreamFields().iterator()), testFieldNode(VALUE));
    assertEquals(2, getIterableSize(dataGrid.getAllInNodes()));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(dataGrid.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(dataGrid.getAllOutNodes()));
    // ---------- mapping - calc checksum
    verifyNodes(IteratorUtils.toList(calcChecksum.getPreviousSteps().iterator()), testLineageNode(dataGrid));
    verifyNodes(IteratorUtils.toList(calcChecksum.getNextSteps().iterator()), testLineageNode(writeToLogChecksum), testLineageNode(writeToLogCalcChecksum));
    verifyNodes(IteratorUtils.toList(calcChecksum.getInputStreamFields().iterator()), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(calcChecksum.getOutputStreamFields().iterator()));
    assertEquals(4, getIterableSize(calcChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(calcChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(calcChecksum.getAllOutNodes()));
    assertEquals(1, getIterableSize(calcChecksum.getOutNodes(LINK_EXECUTES)));
    // ---------- Write to log - mapping - calc checksum
    verifyNodes(IteratorUtils.toList(writeToLogCalcChecksum.getOutputStreamFields().iterator()));
    // verify properties
    verifyNodeProperties(calcChecksum, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] description,input [1] update field names,output [1],output" + " [1] description,output [1] update field names").put("input [1]", "Data grid > [sub] Input checksum").put("input [1] update field names", "false").put("output [1]", "[sub] output checksum > Write to log Checksum").put("output [1] update field names", "false").build());
    // ---------- Write to log Checksum
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksum.getName()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getNextSteps().iterator()));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(writeToLogChecksum.getOutputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyStepIOLinks(writeToLogChecksum, testLineageLink(testFieldNode(CHECKSUM), LINK_DERIVES, testFieldNode(CHECKSUM)), testLineageLink(testFieldNode(VALUE), LINK_DERIVES, testFieldNode(VALUE)));
    assertEquals(5, getIterableSize(writeToLogChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(writeToLogChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(writeToLogChecksum.getAllOutNodes()));
    // ---------- output checksum
    verifyNodes(IteratorUtils.toList(outputChecksum.getPreviousSteps().iterator()), testStepNode(calcChecksumSubTrans.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getNextSteps().iterator()), testStepNode(writeToLogOutputChecksum.getName()));
    verifyNodes(IteratorUtils.toList(outputChecksum.getInputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    verifyNodes(IteratorUtils.toList(outputChecksum.getOutputStreamFields().iterator()), testFieldNode(CHECKSUM), testFieldNode(VALUE));
    assertEquals(5, getIterableSize(outputChecksum.getAllInNodes()));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(outputChecksum.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(outputChecksum.getAllOutNodes()));
    // Verify the following link chains
    // - chain 1: Data grid > outputs > value >  inputs > Input checksum > outputs > value > inputs >
    // calc checksum > outputs > value > input > output checksum > outputs > checksum > outputs > value > inputs >
    // Write to log Checksum > outputs > value
    // - chain 2: Data grid > outputs > value > derives > Input Checksum:value > derives
    // > calc checksum:value > derives > output checksum:value > derives > Write to log Checksum:value
    final FramedMetaverseNode dataGrid_output_value = verifyLinkedNode(dataGrid, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode inputChecksum_output_value = verifyLinkedNode(inputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode calcChecksum_output_value = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode outputChecksum_output_value = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode writeToLogChecksum_output_value = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, VALUE);
    assertEquals(inputChecksum, verifyLinkedNode(dataGrid_output_value, LINK_INPUTS, inputChecksum.getName()));
    assertEquals(calcChecksumSubTrans, verifyLinkedNode(inputChecksum_output_value, LINK_INPUTS, calcChecksumSubTrans.getName()));
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_value, LINK_INPUTS, writeToLogChecksum.getName()));
    assertTrue(IteratorUtils.toList(dataGrid_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(inputChecksum_output_value));
    assertTrue(IteratorUtils.toList(inputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(calcChecksum_output_value));
    assertTrue(IteratorUtils.toList(calcChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(outputChecksum_output_value));
    assertTrue(IteratorUtils.toList(outputChecksum_output_value.getOutNodes(LINK_DERIVES).iterator()).contains(writeToLogChecksum_output_value));
    // Verify the following link chains
    // - chain 1: calc checksum > outputs > checksum > inputs > output > output checksum > inputs > Write to log
    // Checksum > outputs > checksum
    // - chain 2: calc checksum:checksum > derives > output checksum:checksum > derives > Write to log
    // Checksum:checksum
    final FramedMetaverseNode calcChecksum_output_checksum = verifyLinkedNode(calcChecksumSubTrans, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode outputChecksum_output_checksum = verifyLinkedNode(outputChecksum, LINK_OUTPUTS, CHECKSUM);
    final FramedMetaverseNode writeToLogChecksum_output_checksum = verifyLinkedNode(writeToLogChecksum, LINK_OUTPUTS, CHECKSUM);
    assertEquals(writeToLogChecksum, verifyLinkedNode(outputChecksum_output_checksum, LINK_INPUTS, writeToLogChecksum.getName()));
    assertTrue(IteratorUtils.toList(outputChecksum_output_checksum.getOutNodes(LINK_DERIVES).iterator()).contains(writeToLogChecksum_output_checksum));
}

Example 3 with TransformationNode

use of org.pentaho.metaverse.frames.TransformationNode in project pentaho-metaverse by pentaho.

the class SimpleMappingAnalyzerValidationIT method testMoreThanOneIOstepsMappingOut.

@Test
public void testMoreThanOneIOstepsMappingOut() throws Exception {
    final String transNodeName = "simple_moreThanOneIOstepsMappingOut";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode("simple_sub", true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 21, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 53, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, "simple_sub" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { RANDOM_VALUE, RANDOM_VALUE, VALUE, VALUE, VALUE, PARITY, PARITY, NEW_PARITY })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Generate random integer", "Simple mapping (sub-transformation)", "Write to log Parity" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Input parity", "output parity", "calc parity" }, false);
    final TransformationStepNode generateRandomInt = (TransformationStepNode) parentStepNodeMap.get("Generate random integer");
    final TransformationStepNode simpleMapping = (TransformationStepNode) parentStepNodeMap.get("Simple mapping (sub-transformation)");
    final TransformationStepNode writeToLogParity = (TransformationStepNode) parentStepNodeMap.get("Write to log Parity");
    // virtual sub-trans nodes within the parent graph
    final TransformationStepNode inputParity = (TransformationStepNode) subTransStepNodeMap.get("Input parity");
    final TransformationStepNode calcParity = (TransformationStepNode) subTransStepNodeMap.get("calc parity");
    final TransformationStepNode outputParity = (TransformationStepNode) subTransStepNodeMap.get("output parity");
    // ---------- Generate Random Int
    verifyNodes(IteratorUtils.toList(generateRandomInt.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(generateRandomInt.getNextSteps().iterator()), testLineageNode(simpleMapping));
    verifyNodes(IteratorUtils.toList(generateRandomInt.getOutputStreamFields().iterator()), testFieldNode(RANDOM_VALUE, false));
    assertEquals(2, getIterableSize(generateRandomInt.getAllInNodes()));
    assertEquals(1, getIterableSize(generateRandomInt.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(generateRandomInt.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(generateRandomInt.getAllOutNodes()));
    // ---------- "Simple mapping (sub-transformation)
    verifyNodes(IteratorUtils.toList(simpleMapping.getPreviousSteps().iterator()), testLineageNode(generateRandomInt));
    verifyNodes(IteratorUtils.toList(simpleMapping.getInputStreamFields().iterator()), testFieldNode(RANDOM_VALUE, false));
    verifyNodes(IteratorUtils.toList(simpleMapping.getOutputStreamFields().iterator()));
    assertEquals(4, getIterableSize(simpleMapping.getAllInNodes()));
    assertEquals(1, getIterableSize(simpleMapping.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(simpleMapping.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(simpleMapping.getAllOutNodes()));
    assertEquals(1, getIterableSize(simpleMapping.getOutNodes(LINK_EXECUTES)));
    // verify properties
    verifyNodeProperties(simpleMapping, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(PROPERTY_PATH, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put(PROPERTY_VERBOSE_DETAILS, "input [1],input [1] update field names,input [1] rename [1],output [1],output [1] " + "update field names,output [1] rename [1]").put("input [1]", "Generate random integer > [simple_sub] Input parity").put("input [1] update field names", "true").put("input [1] rename [1]", "randomValue > value").put("output [1]", "[simple_sub] output parity > Write to log Parity").put("output [1] rename [1]", "parity > newParity").put("output [1] update field names", "false").build());
    // ---------- output parity
    verifyNodes(IteratorUtils.toList(outputParity.getPreviousSteps().iterator()), testStepNode(calcParity.getName(), false));
    verifyNodes(IteratorUtils.toList(outputParity.getNextSteps().iterator()));
    verifyNodes(IteratorUtils.toList(outputParity.getInputStreamFields().iterator()), testFieldNode(PARITY, false), testFieldNode(VALUE, false));
    verifyNodes(IteratorUtils.toList(outputParity.getOutputStreamFields().iterator()), testFieldNode(PARITY, false), testFieldNode(VALUE, false));
    assertEquals(5, getIterableSize(outputParity.getAllInNodes()));
    assertEquals(1, getIterableSize(outputParity.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(outputParity.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(2, getIterableSize(outputParity.getAllOutNodes()));
    // Verify the following link chains
    // - chain 1: Generate Random Int > outputs > randomValue >  inputs > Input parity > outputs > value > inputs >
    // calc parity > outputs > value > inputs > output parity > outputs > value > inputs
    // > Write to log Parity > outputs > randomValue
    // - chain 2: Generate Random Int > outputs > randomValue > derives > Input Parity:value > derives
    // > calc parity:value > derives > output parity:value > derives > Write to log Parity:randomValue
    final FramedMetaverseNode generateRandomInt_output_randomValue = verifyLinkedNode(generateRandomInt, LINK_OUTPUTS, RANDOM_VALUE);
    final FramedMetaverseNode inputParity_output_value = verifyLinkedNode(inputParity, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode calcParity_output_value = verifyLinkedNode(calcParity, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode outputParity_output_value = verifyLinkedNode(outputParity, LINK_OUTPUTS, VALUE);
    final FramedMetaverseNode writeToLogParity_output_randomValue = verifyLinkedNode(writeToLogParity, LINK_OUTPUTS, RANDOM_VALUE);
    assertEquals(inputParity, verifyLinkedNode(generateRandomInt_output_randomValue, LINK_INPUTS, inputParity.getName()));
    assertEquals(calcParity, verifyLinkedNode(inputParity_output_value, LINK_INPUTS, calcParity.getName()));
    assertEquals(outputParity, verifyLinkedNode(calcParity_output_value, LINK_INPUTS, outputParity.getName()));
    assertEquals(writeToLogParity, verifyLinkedNode(outputParity_output_value, LINK_INPUTS, writeToLogParity.getName()));
    assertEquals(inputParity_output_value, verifyLinkedNode(generateRandomInt_output_randomValue, LINK_DERIVES, VALUE));
    assertEquals(calcParity_output_value, verifyLinkedNode(inputParity_output_value, LINK_DERIVES, VALUE));
    assertEquals(outputParity_output_value, verifyLinkedNode(calcParity_output_value, LINK_DERIVES, VALUE));
    assertEquals(writeToLogParity_output_randomValue, verifyLinkedNode(outputParity_output_value, LINK_DERIVES, RANDOM_VALUE));
    // Verify the following link chains
    // - chain 1: calc parity > outputs > parity > inputs > output parity > outputs > parity > inputs
    // > Write to log Parity > outputs > newParity
    // - chain 2: calc parity:parity > derives > output parity:parity > derives > Write to log
    // Parity:newParity
    final FramedMetaverseNode calcParity_output_parity = verifyLinkedNode(calcParity, LINK_OUTPUTS, PARITY);
    final FramedMetaverseNode outputParity_output_parity = verifyLinkedNode(outputParity, LINK_OUTPUTS, PARITY);
    final FramedMetaverseNode writeToLogParity_output_newParity = verifyLinkedNode(writeToLogParity, LINK_OUTPUTS, NEW_PARITY);
    assertEquals(outputParity, verifyLinkedNode(calcParity_output_parity, LINK_INPUTS, outputParity.getName()));
    assertEquals(writeToLogParity, verifyLinkedNode(outputParity_output_parity, LINK_INPUTS, writeToLogParity.getName()));
    assertEquals(outputParity_output_parity, verifyLinkedNode(calcParity_output_parity, LINK_DERIVES, PARITY));
    assertEquals(writeToLogParity_output_newParity, verifyLinkedNode(outputParity_output_parity, LINK_DERIVES, NEW_PARITY));
}

Example 4 with TransformationNode

use of org.pentaho.metaverse.frames.TransformationNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTransformationStepNodes.

@Test
public void testTransformationStepNodes() throws Exception {
    for (TransformationNode transNode : root.getTransformations()) {
        TransMeta tm = new TransMeta(new FileInputStream(transNode.getPath()), null, true, null, null);
        List<StepMeta> transMetaSteps = tm.getSteps();
        int matchCount = 0;
        for (StepMeta transMetaStep : transMetaSteps) {
            // let's see if the steps are in the graph for this transformation
            TransformationStepNode stepNode = transNode.getStepNode(transMetaStep.getName());
            assertNotNull(stepNode);
            assertEquals("Incorrect type", DictionaryConst.NODE_TYPE_TRANS_STEP, stepNode.getType());
            assertEquals("Incorrect entity type", DictionaryConst.NODE_TYPE_TRANS_STEP, stepNode.getEntity().getName());
            ++matchCount;
        }
        assertEquals("Not all transformation steps are modeled in the graph for [" + tm.getName() + "]", transMetaSteps.size(), matchCount);
        Collection<String> expectedStepNames = new TreeSet<>(transMetaSteps.stream().map(sm -> sm.getName()).collect(Collectors.toList()));
        Collection<String> actualStepNames = new TreeSet<>(StreamSupport.stream(transNode.getStepNodes().spliterator(), false).map(tsn -> tsn.asVertex().getProperty("name").toString()).collect(Collectors.toList()));
        assertEquals("Incorrect number of Steps in the graph for transformation [" + tm.getName() + "]", expectedStepNames, actualStepNames);
    }
}

Example 5 with TransformationNode

use of org.pentaho.metaverse.frames.TransformationNode in project pentaho-metaverse by pentaho.

the class JMSConsumerStepAnalyzerValidationIT method testOneOIWithMappings.

@Test
public void testOneOIWithMappings() throws Exception {
    final String transNodeName = "parent-jms-consumer";
    final String subTransNodeName = "sub-trans-jms-consumer";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true);
    // assert that nodes/edges have expected quantities
    assertEquals("Unexpected number of nodes", 25, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 64, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(transNodeName, subTransNodeName), NODE_TYPE_TRANS_FIELD, Arrays.asList("message", "message", "message", "message", "message", "message", "message_1", "message_1", "message_1", "message_1", "destination", "destination")));
    // verify individual step nodes in parent
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "JMS Consumer", "Write to log" }, false);
    // Verify individual step nodes in subtrans
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Get records from stream", "Sort rows", "Group by", "Write to log" }, false);
    final TransformationStepNode jmsConsumer = (TransformationStepNode) parentStepNodeMap.get("JMS Consumer");
    final TransformationStepNode writeLog = (TransformationStepNode) parentStepNodeMap.get("Write to log");
    // verify that JMS Consumer node has proper outputs and correct type nodes associated with it
    verifyNodes(IteratorUtils.toList(jmsConsumer.getNextSteps().iterator()), testStepNode(writeLog.getName()));
    verifyNodes(IteratorUtils.toList(jmsConsumer.getOutputStreamFields().iterator()), testFieldNode("message", false), testFieldNode("message_1", false));
    assertEquals(1, getIterableSize(jmsConsumer.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(jmsConsumer.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(4, getIterableSize(jmsConsumer.getAllOutNodes()));
    assertEquals(1, getIterableSize(jmsConsumer.getOutNodes(LINK_EXECUTES)));
}