Examples with StreamFieldNode org.pentaho.metaverse.frames.StreamFieldNode used on opensource projects

Example 36 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTextFileInputStep_filenameFromField.

@Test
public void testTextFileInputStep_filenameFromField() throws Exception {
    // this is testing a specific TextFileInputStep instance
    FileInputStepNode fileInputStepNode = root.getTextFileInputStepNode_filenameFromField();
    assertNotNull(fileInputStepNode);
    int countUses = getIterableSize(fileInputStepNode.getFileFieldNodesUses());
    int countInputs = getIterableSize(fileInputStepNode.getInputStreamFields());
    assertEquals(1, countUses);
    int fileFieldCount = 0;
    Iterable<StreamFieldNode> outFields = fileInputStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (!outField.getName().equals("filename")) {
            FieldNode fieldPopulatesMe = outField.getFieldPopulatesMe();
            assertNotNull(fieldPopulatesMe);
            assertEquals(DictionaryConst.NODE_TYPE_FILE_FIELD, fieldPopulatesMe.getType());
            assertEquals(fileInputStepNode, fieldPopulatesMe.getStepThatInputsMe());
            fileFieldCount++;
        }
    }
    // we should have one more input than file fields since we are reading it off of the input stream
    assertEquals(countInputs - 1, fileFieldCount);
    assertEquals(countOutputs, fileFieldCount);
    String filenameField = null;
    TransMeta tm = new TransMeta(new FileInputStream(fileInputStepNode.getTransNode().getPath()), null, true, null, null);
    for (StepMeta stepMeta : tm.getSteps()) {
        if (stepMeta.getName().equals(fileInputStepNode.getName())) {
            org.pentaho.di.trans.steps.fileinput.text.TextFileInputMeta meta = (org.pentaho.di.trans.steps.fileinput.text.TextFileInputMeta) getBaseStepMetaFromStepMeta(stepMeta);
            assertTrue(meta.isAcceptingFilenames());
            filenameField = meta.getAcceptingField();
            assertNotNull(filenameField);
            assertEquals(filenameField, fileInputStepNode.getFileFieldNodesUses().iterator().next().getName());
            // this was the one we cared about...
            break;
        }
    }
}

Example 37 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testCsvFileInputStep.

@Test
public void testCsvFileInputStep() throws Exception {
    // this is testing a specific CsvFileInputStep instance
    CsvFileInputStepNode csvFileInputStepNode = root.getCsvFileInputStepNode();
    assertNotNull(csvFileInputStepNode);
    Iterable<FramedMetaverseNode> inputFiles = csvFileInputStepNode.getInputFiles();
    int countInputFiles = getIterableSize(inputFiles);
    int countOutputs = getIterableSize(csvFileInputStepNode.getOutputStreamFields());
    assertEquals(1, countInputFiles);
    assertEquals(10, countOutputs);
    for (FramedMetaverseNode inputFile : inputFiles) {
        assertTrue(inputFile.getName().endsWith("customers-100.txt"));
    }
    assertEquals("CSV file input", csvFileInputStepNode.getStepType());
    int fileFieldCount = 0;
    Iterable<StreamFieldNode> outFields = csvFileInputStepNode.getOutputStreamFields();
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        FieldNode fieldPopulatesMe = outField.getFieldPopulatesMe();
        assertNotNull(fieldPopulatesMe);
        assertEquals(DictionaryConst.NODE_TYPE_FILE_FIELD, fieldPopulatesMe.getType());
        assertEquals(csvFileInputStepNode, fieldPopulatesMe.getStepThatInputsMe());
        fileFieldCount++;
    }
    assertEquals(countOutputs, fileFieldCount);
}

Example 38 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testRestClientStepNode.

@Test
public void testRestClientStepNode() throws Exception {
    RestClientStepNode node = root.getRestClientStepNode("REST Client");
    assertNotNull(node);
    Iterable<FramedMetaverseNode> inputUrls = node.getInputUrls();
    int countInputUrls = getIterableSize(inputUrls);
    assertEquals(1, countInputUrls);
    assertEquals("REST client", node.getStepType());
    RestMeta stepMeta = (RestMeta) getStepMeta(node);
    for (FramedMetaverseNode inputUrl : inputUrls) {
        assertEquals(stepMeta.getUrl(), inputUrl.getName());
    }
    // check the param  field is "used"
    Iterable<StreamFieldNode> streamFieldNodesUses = node.getStreamFieldNodesUses();
    assertEquals(1, getIterableSize(streamFieldNodesUses));
    for (StreamFieldNode streamFieldNodesUse : streamFieldNodesUses) {
        assertEquals(stepMeta.getParameterField()[0], streamFieldNodesUse.getName());
    }
    Iterable<StreamFieldNode> outputs = node.getOutputStreamFields();
    assertEquals(3, getIterableSize(outputs));
    Iterable<StreamFieldNode> inputs = node.getInputStreamFields();
    assertEquals(2, getIterableSize(inputs));
    for (StreamFieldNode in : inputs) {
        assertNotNull(in.getFieldNodesDerivedFromMe());
        assertEquals(in.getName(), in.getFieldNodesDerivedFromMe().iterator().next().getName());
    }
}

Example 39 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MdiValidationIT method verifyInjectorTextFileOutputNode.

private void verifyInjectorTextFileOutputNode(final TransformationStepNode mdiNode, final String outputFileName, final String[] expectedOutputFieldNameArray, final TransformationStepNode templateReadFromStepNode) {
    final List<TransformationStepNode> mdiHopsToNodes = IteratorUtils.toList(mdiNode.getNextSteps().iterator());
    assertEquals(1, mdiHopsToNodes.size());
    // verify that the text file output step writes to the correct output file
    final TransformationStepNode textFileOutputNode = mdiHopsToNodes.get(0);
    assertEquals("Text file output", textFileOutputNode.getName());
    final List<FileNode> writesToNodes = IteratorUtils.toList(textFileOutputNode.getWritesToFileNodes().iterator());
    assertEquals(1, writesToNodes.size());
    final FileNode fileNode = writesToNodes.get(0);
    assertTrue(String.format("Path expected to end with '%s': %s", outputFileName, fileNode.getPath()), fileNode.getPath().endsWith(outputFileName));
    // verify that the mdi node output fields are inputs into the text file output step
    final List<StreamFieldNode> mdiOutputFields = IteratorUtils.toList(mdiNode.getOutputStreamFields().iterator());
    final List<StreamFieldNode> mdiInputFields = IteratorUtils.toList(mdiNode.getInputStreamFields().iterator());
    final List<StreamFieldNode> inputFields = IteratorUtils.toList(textFileOutputNode.getInputStreamFields().iterator());
    assertEquals(mdiOutputFields.size(), inputFields.size());
    final List<StreamFieldNode> tempalteReadFromStepOutputFields = templateReadFromStepNode == null ? new ArrayList() : IteratorUtils.toList(templateReadFromStepNode.getOutputStreamFields().iterator());
    for (final StreamFieldNode field : inputFields) {
        assertTrue(mdiOutputFields.contains(field));
        // template node that is being read and that the same field is also an input into the mdi step node
        if (templateReadFromStepNode != null) {
            // there should be 1 field that derives this field, and it should be one of the mdi input fields, as well as
            // one of the output fields from the template step that is being read from
            assertEquals(1, getIterableSize(field.getFieldNodesThatDeriveMe()));
            final StreamFieldNode derivingField = (StreamFieldNode) IteratorUtils.toList(field.getFieldNodesThatDeriveMe().iterator()).get(0);
            assertTrue(mdiInputFields.contains(derivingField));
            assertTrue(tempalteReadFromStepOutputFields.contains(derivingField));
            assertEquals(field.getName(), derivingField.getName());
        }
    }
    // verify that the text file output step has the expected output fields, and that each field is derived from an
    // mdi output field with the same name
    final List<StreamFieldNode> outputFields = IteratorUtils.toList(textFileOutputNode.getOutputStreamFields().iterator());
    final List<String> expectedOutputFieldNames = Arrays.asList(expectedOutputFieldNameArray == null ? new String[] {} : expectedOutputFieldNameArray);
    assertEquals(expectedOutputFieldNames.size(), outputFields.size());
    for (final StreamFieldNode field : outputFields) {
        assertTrue(expectedOutputFieldNames.contains(field.getName()));
        // there should be 1 field that derives this field, and it should be one of the mdi output fields
        assertEquals(1, getIterableSize(field.getFieldNodesThatDeriveMe()));
        final StreamFieldNode derivingField = (StreamFieldNode) IteratorUtils.toList(field.getFieldNodesThatDeriveMe().iterator()).get(0);
        assertTrue(mdiOutputFields.contains(derivingField));
        assertEquals(field.getName(), derivingField.getName());
    }
}