Examples with TransformationStepNode org.pentaho.metaverse.frames.TransformationStepNode used on opensource projects

Example 16 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MdiValidationIT method testMdiInjectorNoStreaMapTwoSteps.

@Test
public void testMdiInjectorNoStreaMapTwoSteps() throws Exception {
    final String transNodeName = "injector_no_stream_map_2_steps";
    final String subTransNodeName = "template_no_stream_map_2_steps";
    initTest(transNodeName);
    final TransformationNode injectorTransNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 38, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 99, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "File Name", "Separator", "Field Name", "Trim Type", "First Name", "First Name", "First Name", "Last Name", "Last Name", "Last Name", "Age", "Age", "Age" }), NODE_TYPE_STEP_PROPERTY, Arrays.asList(new String[] { "FILENAME", "SEPARATOR", "SEPARATOR", "OUTPUT_FIELDNAME", "OUTPUT_FIELDNAME", "OUTPUT_TRIM" })));
    // The injector should "contain" 4 step nodes and no virtual nodes
    final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(injectorTransNode, new String[] { "Text Output - Fields", "Text Output", "ETL Metadata Injection", "Text file output" }, false);
    // the template subTransformation should "contain" no step nodes (non-virtual) and one virtual nodes
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "My Text file output", "My Text file output [2]", "My Generate Rows" }, false);
    final TransformationStepNode textOutputFieldsNode = (TransformationStepNode) stepNodeMap.get("Text Output - Fields");
    final TransformationStepNode textOutputNode = (TransformationStepNode) stepNodeMap.get("Text Output");
    final TransformationStepNode metaInject = (TransformationStepNode) stepNodeMap.get("ETL Metadata Injection");
    final TransformationStepNode textFileOutput = (TransformationStepNode) stepNodeMap.get("Text file output");
    final TransformationStepNode myTextFileOutputNode = (TransformationStepNode) subTransStepNodeMap.get("My Text file output");
    final TransformationStepNode myTextFileOutputNode2 = (TransformationStepNode) subTransStepNodeMap.get("My Text file output [2]");
    // Text Output
    verifyNodes(IteratorUtils.toList(textOutputNode.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getNextSteps().iterator()), testStepNode(metaInject.getName()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getInputStreamFields().iterator()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getOutputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"));
    assertEquals(2, getIterableSize(textOutputNode.getAllInNodes()));
    assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(textOutputNode.getAllOutNodes()));
    assertEquals(1, getIterableSize(textOutputNode.getOutNodes(LINK_HOPSTO)));
    // ETL Metadata injection
    // verify the mdi node and connections to the node - since we are not streaming, the mdi node should have no
    // output fields
    assertEquals(metaInject, verifyMdiNode(stepNodeMap, subTransNode, new String[] {}));
    verifyInjectorTextFileOutputNode(metaInject, "injector_no_stream_map_2_steps.txt", new String[] {}, myTextFileOutputNode);
    // verify all the expected mdi properties and links from fields to properties exist
    verifyMdiInputs(metaInject, textOutputNode, myTextFileOutputNode.getName(), ImmutableMap.of("File Name", "FILENAME", "Separator", "SEPARATOR"));
    verifyMdiInputs(metaInject, textOutputFieldsNode, myTextFileOutputNode.getName(), ImmutableMap.of("Field Name", "OUTPUT_FIELDNAME", "Trim Type", "OUTPUT_TRIM"));
    verifyNodes(IteratorUtils.toList(metaInject.getPreviousSteps().iterator()), testStepNode(textOutputNode.getName()), testStepNode(textOutputFieldsNode.getName()));
    verifyNodes(IteratorUtils.toList(metaInject.getNextSteps().iterator()), testStepNode(textFileOutput.getName()));
    verifyNodes(IteratorUtils.toList(metaInject.getInputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type"));
    verifyNodes(IteratorUtils.toList(metaInject.getStreamFieldNodesUses().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type"));
    verifyNodes(IteratorUtils.toList(metaInject.getOutputStreamFields().iterator()));
    assertEquals(8, getIterableSize(metaInject.getAllInNodes()));
    assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(6, getIterableSize(metaInject.getAllOutNodes()));
    assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_HOPSTO)));
    assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_EXECUTES)));
    // Verify the following chains:
    // Chain 1: Text Output > outputs > First Name > populates > My Text file output:FILENAME
    // Chain 2: My Text file output > contains > FILENAME
    // Chain 3: Text Output > outputs > Separator > populates > My Text file output:SEPARATOR (x2)
    final FramedMetaverseNode textOutput_output_fileName = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "File Name");
    final FramedMetaverseNode myTextFileOutput_property_fileName = verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "FILENAME");
    assertEquals(myTextFileOutput_property_fileName, verifyLinkedNode(textOutput_output_fileName, LINK_POPULATES, "FILENAME"));
    final FramedMetaverseNode textOutput_output_separator = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "Separator");
    final FramedMetaverseNode myTextFileOutput_property_separator = verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "SEPARATOR");
    final List<FramedMetaverseNode> textOutput_property_separators = verifyLinkedNodes(textOutput_output_separator, LINK_POPULATES, "SEPARATOR");
    assertEquals(2, textOutput_property_separators.size());
    assertTrue(textOutput_property_separators.contains(myTextFileOutput_property_separator));
    // Verify the following chains:
    // Chain 1: Text Output - Fields > outputs > First Name > populates > My Text file output [2]:OUTPUT_FIELDNAME (x2)
    // Chain 2: My Text file output [2] > contains > OUTPUT_FIELDNAME
    final FramedMetaverseNode textOutputFields_output_fieldName = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Field Name");
    final FramedMetaverseNode myTextFileOutput_property_fieldName = verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "OUTPUT_FIELDNAME");
    final List<FramedMetaverseNode> textOutputFields_property_fieldNames = verifyLinkedNodes(textOutputFields_output_fieldName, LINK_POPULATES, "OUTPUT_FIELDNAME");
    assertEquals(2, textOutputFields_property_fieldNames.size());
    assertTrue(textOutputFields_property_fieldNames.contains(myTextFileOutput_property_fieldName));
    // validate properties
    verifyNodeProperties(metaInject, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put("runResultingTransformation", "true").put("targetFile", SKIP).put(PROPERTY_VERBOSE_DETAILS, "mapping [1],mapping [2],mapping [3],mapping [4],mapping [5],mapping [6]").put("mapping [1]", "Text Output: Separator > [template_no_stream_map_2_steps] My Text file output: SEPARATOR").put("mapping [2]", "Text Output: Separator > [template_no_stream_map_2_steps] My Text file output [2]: " + "SEPARATOR").put("mapping [3]", "Text Output - Fields: Trim Type > [template_no_stream_map_2_steps] My Text file output: " + "OUTPUT_TRIM").put("mapping [4]", "Text Output - Fields: Field Name > [template_no_stream_map_2_steps] My Text file output: " + "OUTPUT_FIELDNAME").put("mapping [5]", "Text Output: File Name > [template_no_stream_map_2_steps] My Text file output: FILENAME").put("mapping [6]", "Text Output - Fields: Field Name > [template_no_stream_map_2_steps] My Text file output " + "[2]: OUTPUT_FIELDNAME").build());
}

Example 17 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MdiValidationIT method testMdiInjectorStreamReadingAndStreamingSameStep.

@Test
public void testMdiInjectorStreamReadingAndStreamingSameStep() throws Exception {
    final String transNodeName = "injector_stream_same_as_mapping";
    final String subTransNodeName = "template_stream_same_as_mapping";
    initTest(transNodeName);
    final TransformationNode injectorTransNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 35, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 96, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "File Name", "Separator", "Field Name", "Type", "Trim Type", "Dummy", "First Name", "First Name", "First Name", "First Name", "Last Name", "Last Name", "Last Name", "Last Name", "Age", "Age" }), NODE_TYPE_STEP_PROPERTY, Arrays.asList(new String[] { "FILENAME", "SEPARATOR" })));
    // The injector should "contain" 4 step nodes and no virtual nodes
    final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(injectorTransNode, new String[] { "Text Output - Fields", "Text Output", "ETL Metadata Injection", "Text file output" }, false);
    // the template subTransformation should "contain" no step nodes (non-virtual) and one virtual nodes
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "My Text file output", "My Generate Rows" }, false);
    final TransformationStepNode textOutputFieldsNode = (TransformationStepNode) stepNodeMap.get("Text Output - Fields");
    final TransformationStepNode textOutputNode = (TransformationStepNode) stepNodeMap.get("Text Output");
    final TransformationStepNode metaInject = (TransformationStepNode) stepNodeMap.get("ETL Metadata Injection");
    final TransformationStepNode textFileOutput = (TransformationStepNode) stepNodeMap.get("Text file output");
    final TransformationStepNode myTextFileOutputNode = (TransformationStepNode) subTransStepNodeMap.get("My Text file output");
    // Text Output
    verifyNodes(IteratorUtils.toList(textOutputNode.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getNextSteps().iterator()), testStepNode(metaInject.getName()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getInputStreamFields().iterator()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getOutputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"));
    assertEquals(2, getIterableSize(textOutputNode.getAllInNodes()));
    assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(textOutputNode.getAllOutNodes()));
    assertEquals(1, getIterableSize(textOutputNode.getOutNodes(LINK_HOPSTO)));
    // ETL Metadata injection
    // verify the mdi node and connections to the node
    assertEquals(metaInject, verifyMdiNode(stepNodeMap, subTransNode, new String[] { "First Name", "Last Name" }));
    verifyInjectorTextFileOutputNode(metaInject, "stream_same_as_mapping_injector.txt", new String[] { "First Name", "Last Name" }, myTextFileOutputNode);
    // verify all the expected mdi properties and links from fields to properties exist
    verifyMdiInputs(metaInject, textOutputNode, myTextFileOutputNode.getName(), ImmutableMap.of("File Name", "FILENAME", "Separator", "SEPARATOR"));
    verifyMdiInputs(metaInject, textOutputFieldsNode, myTextFileOutputNode.getName(), ImmutableMap.of("Field Name", "", "Trim Type", "", "Type", "", "Dummy", ""));
    verifyNodes(IteratorUtils.toList(metaInject.getPreviousSteps().iterator()), testStepNode(textOutputNode.getName()), testStepNode(textOutputFieldsNode.getName()));
    verifyNodes(IteratorUtils.toList(metaInject.getNextSteps().iterator()), testStepNode(textFileOutput.getName()));
    verifyNodes(IteratorUtils.toList(metaInject.getInputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Type"), testFieldNode("Trim Type"), testFieldNode("Dummy"), testFieldNode("First Name"), testFieldNode("Last Name"), testFieldNode("Age"));
    verifyNodes(IteratorUtils.toList(metaInject.getStreamFieldNodesUses().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type"));
    verifyNodes(IteratorUtils.toList(metaInject.getOutputStreamFields().iterator()), testFieldNode("First Name"), testFieldNode("Last Name"));
    assertEquals(13, getIterableSize(metaInject.getAllInNodes()));
    assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(8, getIterableSize(metaInject.getAllOutNodes()));
    assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_HOPSTO)));
    assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_EXECUTES)));
    // Verify the following chains:
    // Chain 1: Text Output > outputs First Name > populates > FILENAME
    // Chain 2: My Text file output > contains > FILENAME
    final FramedMetaverseNode textOutput_output_fileName = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "File Name");
    final FramedMetaverseNode textOutput_property_fileName = verifyLinkedNode(textOutput_output_fileName, LINK_POPULATES, "FILENAME");
    assertEquals(textOutput_property_fileName, verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "FILENAME"));
    // Verify the following chains:
    // - Text Output - Fields > outputs > Field Name > derives > My Text file output:First Name > inputs > ETL Metadata
    // injection
    // - Text Output - Fields > outputs > Type > derives > My Text file output:Last Name > inputs > ETL Metadata
    // injection
    // - Text Output - Fields > outputs > Trim Type > derives > My Text file output:Age > inputs > ETL Metadata
    // injection
    final FramedMetaverseNode textOutputFields_output_fieldName = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Field Name");
    final FramedMetaverseNode myTextFileOutput_output_firstName = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "First Name");
    assertEquals(myTextFileOutput_output_firstName, verifyLinkedNode(textOutputFields_output_fieldName, LINK_DERIVES, "First Name"));
    assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_INPUTS, metaInject.getName()));
    final FramedMetaverseNode textOutputFields_output_type = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Type");
    final FramedMetaverseNode myTextFileOutput_output_lastName = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "Last Name");
    assertEquals(myTextFileOutput_output_lastName, verifyLinkedNode(textOutputFields_output_type, LINK_DERIVES, "Last Name"));
    assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_lastName, LINK_INPUTS, metaInject.getName()));
    final FramedMetaverseNode textOutputFields_output_trimType = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Trim Type");
    final FramedMetaverseNode myTextFileOutput_output_age = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "Age");
    assertEquals(myTextFileOutput_output_age, verifyLinkedNode(textOutputFields_output_trimType, LINK_DERIVES, "Age"));
    assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_age, LINK_INPUTS, metaInject.getName()));
    // Verify the following chains:
    // - My Text file output:First Name > derives > ETL Metadata injection:First Name > inputs > Text file output >
    // outputs First Name
    final FramedMetaverseNode metaInject_output_firstName = verifyLinkedNode(metaInject, LINK_OUTPUTS, "First Name");
    assertEquals(metaInject_output_firstName, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_DERIVES, "First Name"));
    assertEquals(textFileOutput, verifyLinkedNode(metaInject_output_firstName, LINK_INPUTS, textFileOutput.getName()));
    // validate properties
    verifyNodeProperties(metaInject, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put("runResultingTransformation", "true").put("streamTargetStepname", SKIP).put("streamSourceStepname", SKIP).put("targetFile", SKIP).put("sourceStepName", SKIP).put(PROPERTY_VERBOSE_DETAILS, "mapping [1],ignored mapping [1],mapping [2],ignored mapping [2]").put("mapping [1]", "Text Output: Separator > [template_stream_same_as_mapping] My Text file output: SEPARATOR").put("mapping [2]", "Text Output: File Name > [template_stream_same_as_mapping] My Text file output: FILENAME").put("ignored mapping [1]", "Text Output - Fields: Trim Type > [template_stream_same_as_mapping] My Text file " + "output: OUTPUT_TRIM").put("ignored mapping [2]", "Text Output - Fields: Field Name > [template_stream_same_as_mapping] My Text file" + " output: OUTPUT_FIELDNAME").build());
}

Example 18 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class MdiValidationIT method testMdiInjectorStreamReadingAndStreamingDifferentSteps.

@Test
public void testMdiInjectorStreamReadingAndStreamingDifferentSteps() throws Exception {
    final String transNodeName = "injector_stream_different_than_mapping";
    final String subTransNodeName = "template_stream_different_than_mapping";
    initTest(transNodeName);
    final TransformationNode injectorTransNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 40, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 113, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "File Name", "Separator", "Field Name", "Type", "Trim Type", "Dummy", "First Name", "First Name", "First Name", "First Name", "First Name", "Last Name", "Last Name", "Last Name", "Last Name", "Last Name", "Age", "Age", "Age" }), NODE_TYPE_STEP_PROPERTY, Arrays.asList(new String[] { "FILENAME", "SEPARATOR" })));
    // The injector should "contain" 4 step nodes and no virtual nodes
    final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(injectorTransNode, new String[] { "Text Output - Fields", "Text Output", "ETL Metadata Injection", "Text file output" }, false);
    // the template subTransformation should "contain" no step nodes (non-virtual) and one virtual nodes
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "My Text file output", "My Text file output [2]", "My Generate Rows" }, false);
    final TransformationStepNode textOutputFieldsNode = (TransformationStepNode) stepNodeMap.get("Text Output - Fields");
    final TransformationStepNode textOutputNode = (TransformationStepNode) stepNodeMap.get("Text Output");
    final TransformationStepNode metaInject = (TransformationStepNode) stepNodeMap.get("ETL Metadata Injection");
    final TransformationStepNode textFileOutput = (TransformationStepNode) stepNodeMap.get("Text file output");
    final TransformationStepNode myTextFileOutputNode = (TransformationStepNode) subTransStepNodeMap.get("My Text file output");
    final TransformationStepNode myTextFileOutputNode2 = (TransformationStepNode) subTransStepNodeMap.get("My Text file output [2]");
    // Text Output
    verifyNodes(IteratorUtils.toList(textOutputNode.getPreviousSteps().iterator()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getNextSteps().iterator()), testStepNode(metaInject.getName()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getInputStreamFields().iterator()));
    verifyNodes(IteratorUtils.toList(textOutputNode.getOutputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"));
    assertEquals(2, getIterableSize(textOutputNode.getAllInNodes()));
    assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(textOutputNode.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(3, getIterableSize(textOutputNode.getAllOutNodes()));
    assertEquals(1, getIterableSize(textOutputNode.getOutNodes(LINK_HOPSTO)));
    // ETL Metadata injection
    // verify the mdi node and connections to the node
    assertEquals(metaInject, verifyMdiNode(stepNodeMap, subTransNode, new String[] { "First Name", "Last Name" }));
    verifyInjectorTextFileOutputNode(metaInject, "stream_different_than_mapping_injector.txt", new String[] { "First Name", "Last Name" }, myTextFileOutputNode);
    // verify all the expected mdi properties and links from fields to properties exist
    verifyMdiInputs(metaInject, textOutputNode, myTextFileOutputNode.getName(), ImmutableMap.of("File Name", "FILENAME", "Separator", "SEPARATOR"));
    verifyMdiInputs(metaInject, textOutputFieldsNode, myTextFileOutputNode.getName(), ImmutableMap.of("Field Name", "", "Trim Type", "", "Type", "", "Dummy", ""));
    verifyNodes(IteratorUtils.toList(metaInject.getPreviousSteps().iterator()), testStepNode(textOutputNode.getName()), testStepNode(textOutputFieldsNode.getName()));
    verifyNodes(IteratorUtils.toList(metaInject.getNextSteps().iterator()), testStepNode(textFileOutput.getName()));
    verifyNodes(IteratorUtils.toList(metaInject.getInputStreamFields().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Type"), testFieldNode("Trim Type"), testFieldNode("Dummy"), testFieldNode("First Name"), testFieldNode("Last Name"), testFieldNode("Age"));
    verifyNodes(IteratorUtils.toList(metaInject.getStreamFieldNodesUses().iterator()), testFieldNode("File Name"), testFieldNode("Separator"), testFieldNode("Field Name"), testFieldNode("Trim Type"));
    verifyNodes(IteratorUtils.toList(metaInject.getOutputStreamFields().iterator()), testFieldNode("First Name"), testFieldNode("Last Name"));
    assertEquals(13, getIterableSize(metaInject.getAllInNodes()));
    assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(metaInject.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(8, getIterableSize(metaInject.getAllOutNodes()));
    assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_HOPSTO)));
    assertEquals(1, getIterableSize(metaInject.getOutNodes(LINK_EXECUTES)));
    // Verify the following chains:
    // Chain 1: Text Output > outputs First Name > populates > FILENAME
    // Chain 2: My Text file output > contains > FILENAME
    final FramedMetaverseNode textOutput_output_fileName = verifyLinkedNode(textOutputNode, LINK_OUTPUTS, "File Name");
    final FramedMetaverseNode textOutput_property_fileName = verifyLinkedNode(textOutput_output_fileName, LINK_POPULATES, "FILENAME");
    assertEquals(textOutput_property_fileName, verifyLinkedNode(myTextFileOutputNode, LINK_CONTAINS, "FILENAME"));
    // Verify the following chains:
    // - Text Output - Fields > outputs > Field Name > derives My Text file output [2]: First Name
    // - My Text file output: First Name > inputs > ETL Metadata injection
    // - My Text file output: First Name > derives > ETL Metadata injection:First Name
    final FramedMetaverseNode textOutputFields_output_fieldName = verifyLinkedNode(textOutputFieldsNode, LINK_OUTPUTS, "Field Name");
    final FramedMetaverseNode myTextFileOutput_output_firstName = verifyLinkedNode(myTextFileOutputNode, LINK_OUTPUTS, "First Name");
    final FramedMetaverseNode myTextFileOutput2_output_firstName = verifyLinkedNode(myTextFileOutputNode2, LINK_OUTPUTS, "First Name");
    final FramedMetaverseNode metaInject_output_firstName = verifyLinkedNode(metaInject, LINK_OUTPUTS, "First Name");
    assertEquals(myTextFileOutput2_output_firstName, verifyLinkedNode(textOutputFields_output_fieldName, LINK_DERIVES, "First Name"));
    assertEquals(metaInject, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_INPUTS, metaInject.getName()));
    assertEquals(metaInject_output_firstName, verifyLinkedNode(myTextFileOutput_output_firstName, LINK_DERIVES, "First Name"));
    // validate properties
    verifyNodeProperties(metaInject, new ImmutableMap.Builder<String, Object>().put(PROPERTY_STEP_TYPE, SKIP).put("color", SKIP).put(PROPERTY_PLUGIN_ID, SKIP).put(PROPERTY_TYPE, SKIP).put(PROPERTY_ANALYZER, SKIP).put(PROPERTY_CATEGORY, SKIP).put(PROPERTY_COPIES, SKIP).put(PROPERTY_LOGICAL_ID, SKIP).put(PROPERTY_NAME, SKIP).put(PROPERTY_NAMESPACE, SKIP).put(NODE_VIRTUAL, SKIP).put("subTransformation", SKIP).put("runResultingTransformation", "true").put("streamTargetStepname", SKIP).put("streamSourceStepname", SKIP).put("targetFile", SKIP).put("sourceStepName", SKIP).put(PROPERTY_VERBOSE_DETAILS, "mapping [1],ignored mapping [1],mapping [2],ignored mapping [2]").put("mapping [1]", "Text Output: Separator > [template_stream_different_than_mapping] My Text file output: SEPARATOR").put("mapping [2]", "Text Output: File Name > [template_stream_different_than_mapping] My Text file output: FILENAME").put("ignored mapping [1]", "Text Output - Fields: Trim Type > [template_stream_different_than_mapping]" + " My Text file output: OUTPUT_TRIM").put("ignored mapping [2]", "Text Output - Fields: Field Name > [template_stream_different_than_mapping]" + " My Text file output: OUTPUT_FIELDNAME").build());
}

Example 19 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class SingleThreaderStepAnalyzerValidationIT method testOneOIWithMappings.

@Test
public void testOneOIWithMappings() throws Exception {
    final String transNodeName = "parent-single-threader";
    final String subTransNodeName = "sub-trans";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    final TransformationNode subTransNode = verifyTransformationNode(subTransNodeName, true);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals("Unexpected number of nodes", 30, getIterableSize(framedGraph.getVertices()));
    assertEquals("Unexpected number of edges", 87, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName, subTransNodeName }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "strTest", "strTest", "strTest", "strTest", "strTest", "strTest", "strTest", "strTest", "counter", "counter", "counter", "counter", "counter", "counter", "counter" })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> parentStepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Data grid", "Add constants", "Single threader", "Write to log", "Write to log 2" }, false);
    final Map<String, FramedMetaverseNode> subTransStepNodeMap = verifyTransformationSteps(subTransNode, new String[] { "Mapping input specification", "String operations", "Mapping output specification" }, false);
    final TransformationStepNode singleThreader = (TransformationStepNode) parentStepNodeMap.get("Single threader");
    final TransformationStepNode writeToLog = (TransformationStepNode) parentStepNodeMap.get("Write to log");
    final TransformationStepNode writeToLog2 = (TransformationStepNode) parentStepNodeMap.get("Write to log 2");
    final TransformationStepNode inputSpec = (TransformationStepNode) subTransStepNodeMap.get("Mapping input specification");
    final TransformationStepNode stringOperations = (TransformationStepNode) subTransStepNodeMap.get("String operations");
    final TransformationStepNode outputSpec = (TransformationStepNode) subTransStepNodeMap.get("Mapping output specification");
    // ---------- Generate Random Int
    verifyNodes(IteratorUtils.toList(singleThreader.getPreviousSteps().iterator()), testStepNode("Add constants"));
    verifyNodes(IteratorUtils.toList(singleThreader.getNextSteps().iterator()), testStepNode(writeToLog.getName()), testStepNode(writeToLog2.getName()));
    verifyNodes(IteratorUtils.toList(singleThreader.getInputStreamFields().iterator()), testFieldNode("counter", false), testFieldNode("strTest", false));
    verifyNodes(IteratorUtils.toList(singleThreader.getOutputStreamFields().iterator()), testFieldNode("counter", false), testFieldNode("strTest", false));
    assertEquals(5, getIterableSize(singleThreader.getAllInNodes()));
    assertEquals(1, getIterableSize(singleThreader.getInNodes(LINK_CONTAINS)));
    assertEquals(1, getIterableSize(singleThreader.getInNodes(LINK_TYPE_CONCEPT)));
    assertEquals(5, getIterableSize(singleThreader.getAllOutNodes()));
    assertEquals(1, getIterableSize(singleThreader.getOutNodes(LINK_EXECUTES)));
    // verify the following link chain:
    // Single threader > outputs > counter > derives > Mapping input specification:counter > derives >
    // String operations:counter > Mapping output specification:counter > derives > Single threader:counter
    final FramedMetaverseNode singleThreader_output_counter = verifyLinkedNode(singleThreader, LINK_OUTPUTS, "counter");
    final FramedMetaverseNode inputSpec_output_counter = verifyLinkedNode(inputSpec, LINK_OUTPUTS, "counter");
    final FramedMetaverseNode stringOperations_output_counter = verifyLinkedNode(stringOperations, LINK_OUTPUTS, "counter");
    final FramedMetaverseNode outputSpec_output_counter = verifyLinkedNode(outputSpec, LINK_OUTPUTS, "counter");
    assertEquals(inputSpec_output_counter, verifyLinkedNode(singleThreader_output_counter, LINK_DERIVES, "counter"));
    assertEquals(stringOperations_output_counter, verifyLinkedNode(inputSpec_output_counter, LINK_DERIVES, "counter"));
    assertEquals(outputSpec_output_counter, verifyLinkedNode(stringOperations_output_counter, LINK_DERIVES, "counter"));
    assertEquals(singleThreader_output_counter, verifyLinkedNode(outputSpec_output_counter, LINK_DERIVES, "counter"));
}

Example 20 with TransformationStepNode

use of org.pentaho.metaverse.frames.TransformationStepNode in project pentaho-metaverse by pentaho.

the class TableOutputValidationIT method testTableIoDbFieldRenamedAndListTrimmed.

@Test
public void testTableIoDbFieldRenamedAndListTrimmed() throws Exception {
    final String transNodeName = "table_io_db_field_renamed_and_list_trimmed";
    initTest(transNodeName);
    final TransformationNode transformationNode = verifyTransformationNode(transNodeName, false);
    // smoke test - verify that the right number of nodes and edges exist in the graph and that the expected top
    // level nodes of expected types exist
    assertEquals(28, getIterableSize(framedGraph.getVertices()));
    assertEquals(69, getIterableSize(framedGraph.getEdges()));
    verifyNodesTypes(ImmutableMap.of(NODE_TYPE_TRANS, Arrays.asList(new String[] { transNodeName }), NODE_TYPE_DATASOURCE, Arrays.asList(new String[] { "Postgres - localhost" }), NODE_TYPE_DATA_TABLE, Arrays.asList(new String[] { "people" }), NODE_TYPE_DATA_COLUMN, Arrays.asList(new String[] { "first_name", "first_name", "last_name" }), NODE_TYPE_TRANS_FIELD, Arrays.asList(new String[] { "first_name", "first_name_renamed", "first_name_renamed", "first_name_renamed", "last_name", "last_name", "last_name", "last_name" })));
    // verify individual step nodes
    final Map<String, FramedMetaverseNode> stepNodeMap = verifyTransformationSteps(transformationNode, new String[] { "Table input", "Select values", "Table output", "Dummy (do nothing)" }, false);
    // ---------- Table input
    final TransformationStepNode tableInput = (TransformationStepNode) stepNodeMap.get("Table input");
    verifyNodes(IteratorUtils.toList(tableInput.getReadByNodes().iterator()), testSqlQueryNode("SQL", false));
    verifyNodes(IteratorUtils.toList(tableInput.getNextSteps().iterator()), testStepNode("Select values", false));
    verifyNodes(IteratorUtils.toList(tableInput.getInputFields().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false));
    verifyNodes(IteratorUtils.toList(tableInput.getOutputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false));
    verifyStepIOLinks(tableInput, testLineageLink(testColumnNode("first_name", false), LINK_POPULATES, testFieldNode("first_name", false)), testLineageLink(testColumnNode("last_name", false), LINK_POPULATES, testFieldNode("last_name", false)));
    final FramedMetaverseNode sqlNode = (FramedMetaverseNode) IteratorUtils.toList(tableInput.getReadByNodes().iterator()).get(0);
    verifyNodes(IteratorUtils.toList(sqlNode.getContainedNodes().iterator()), testColumnNode("first_name", false), testColumnNode("last_name", false));
    // ---------- Select values
    final TransformationStepNode selectValues = (TransformationStepNode) stepNodeMap.get("Select values");
    verifyNodes(IteratorUtils.toList(selectValues.getPreviousSteps().iterator()), testStepNode("Table input", false));
    verifyNodes(IteratorUtils.toList(selectValues.getNextSteps().iterator()), testStepNode("Table output", false));
    verifyNodes(IteratorUtils.toList(selectValues.getInputStreamFields().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false));
    verifyNodes(IteratorUtils.toList(selectValues.getOutputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testFieldNode("last_name", false));
    verifyNodes(IteratorUtils.toList(selectValues.getStreamFieldNodesUses().iterator()), testFieldNode("first_name", false), testFieldNode("last_name", false));
    verifyStepIOLinks(selectValues, Arrays.asList(new TestLineageLink[] { testLineageLink(testFieldNode("first_name", false), LINK_DERIVES, testFieldNode("first_name_renamed", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false)) }));
    // ---------- Table output
    final TransformationStepNode tableOutput = (TransformationStepNode) stepNodeMap.get("Table output");
    verifyNodes(IteratorUtils.toList(tableOutput.getPreviousSteps().iterator()), testStepNode("Select values", false));
    verifyNodes(IteratorUtils.toList(tableOutput.getNextSteps().iterator()), testStepNode("Dummy (do nothing)", false));
    verifyNodes(IteratorUtils.toList(tableOutput.getWritesToNodes().iterator()), testTableNode("people", false));
    verifyNodes(IteratorUtils.toList(tableOutput.getInputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testFieldNode("last_name", false));
    verifyNodes(IteratorUtils.toList(tableOutput.getOutputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testColumnNode("first_name", false), testFieldNode("last_name", false));
    verifyStepIOLinks(tableOutput, testLineageLink(testFieldNode("first_name_renamed", false), LINK_DERIVES, testFieldNode("first_name_renamed", false)), testLineageLink(testFieldNode("first_name_renamed", false), LINK_POPULATES, testColumnNode("first_name", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false)));
    final FramedMetaverseNode people = (FramedMetaverseNode) IteratorUtils.toList(tableOutput.getWritesToNodes().iterator()).get(0);
    verifyNodes(IteratorUtils.toList(people.getContainedNodes().iterator()), testColumnNode("first_name", false));
    // ---------- Table output
    final TransformationStepNode dummy = (TransformationStepNode) stepNodeMap.get("Dummy (do nothing)");
    verifyNodes(IteratorUtils.toList(dummy.getPreviousSteps().iterator()), testStepNode("Table output", false));
    verifyNodes(IteratorUtils.toList(tableOutput.getInputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testFieldNode("last_name", false));
    verifyNodes(IteratorUtils.toList(tableOutput.getOutputStreamFields().iterator()), testFieldNode("first_name_renamed", false), testColumnNode("first_name", false), testFieldNode("last_name", false));
    verifyStepIOLinks(tableOutput, testLineageLink(testFieldNode("first_name_renamed", false), LINK_DERIVES, testFieldNode("first_name_renamed", false)), testLineageLink(testFieldNode("last_name", false), LINK_DERIVES, testFieldNode("last_name", false)));
}