Examples with StreamFieldNode org.pentaho.metaverse.frames.StreamFieldNode used on opensource projects

Example 21 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTextFileOutputStepNode.

@Test
public void testTextFileOutputStepNode() throws Exception {
    TextFileOutputStepNode textFileOutputStepNode = root.getTextFileOutputStepNode("textFileOutput", "Text file output");
    TextFileOutputMeta meta = (TextFileOutputMeta) getStepMeta(textFileOutputStepNode);
    TransMeta tm = meta.getParentStepMeta().getParentTransMeta();
    String[] fileNames = meta.getFiles(tm);
    RowMetaInterface incomingFields = tm.getPrevStepFields(meta.getParentStepMeta());
    int outputFields = getExpectedOutputFieldCount(meta);
    assertNotNull(textFileOutputStepNode);
    // should write to one file
    Iterable<FramedMetaverseNode> outputFiles = textFileOutputStepNode.getOutputFiles();
    assertEquals(fileNames.length, getIterableSize(outputFiles));
    int i = 0;
    for (FramedMetaverseNode node : outputFiles) {
        assertEquals(normalizeFilePath(fileNames[i++]), normalizeFilePath(node.getName()));
    }
    Iterable<StreamFieldNode> outFields = textFileOutputStepNode.getOutputStreamFields();
    int outFieldCount = getIterableSize(outFields);
    // should have output stream nodes as well as file nodes
    assertEquals(outputFields + meta.getOutputFields().length, outFieldCount);
    int fileFieldCount = 0;
    for (StreamFieldNode outField : outFields) {
        if (DictionaryConst.NODE_TYPE_FILE_FIELD.equals(outField.getType())) {
            ValueMetaInterface vmi = incomingFields.searchValueMeta(outField.getName());
            assertEquals(vmi.getName(), outField.getFieldPopulatesMe().getName());
            fileFieldCount++;
        }
    }
    assertEquals(fileFieldCount, outFieldCount / 2);
}

Example 22 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testRestClientStepNode_urlFromField.

@Test
public void testRestClientStepNode_urlFromField() throws Exception {
    RestClientStepNode node = root.getRestClientStepNode("REST Client - parameterized");
    assertNotNull(node);
    Iterable<FramedMetaverseNode> inputUrls = node.getInputUrls();
    assertEquals("REST Client", node.getStepType());
    RestMeta stepMeta = (RestMeta) getStepMeta(node);
    for (FramedMetaverseNode inputUrl : inputUrls) {
        assertEquals(stepMeta.getUrlField(), inputUrl.getName());
    }
    Set<String> usedFields = new HashSet<>();
    Collections.addAll(usedFields, stepMeta.getHeaderField());
    Collections.addAll(usedFields, stepMeta.getParameterField());
    if (stepMeta.isUrlInField()) {
        usedFields.add(stepMeta.getUrlField());
    }
    if (stepMeta.isDynamicMethod()) {
        usedFields.add(stepMeta.getMethodFieldName());
    }
    if (StringUtils.isNotEmpty(stepMeta.getBodyField())) {
        usedFields.add(stepMeta.getBodyField());
    }
    // check the param  field is "used"
    Iterable<StreamFieldNode> streamFieldNodesUses = node.getStreamFieldNodesUses();
    assertEquals(usedFields.size(), getIterableSize(streamFieldNodesUses));
    for (StreamFieldNode streamFieldNodesUse : streamFieldNodesUses) {
        assertTrue(usedFields.contains(streamFieldNodesUse.getName()));
    }
    Iterable<StreamFieldNode> outputs = node.getOutputStreamFields();
    assertEquals(getExpectedOutputFieldCount(stepMeta), getIterableSize(outputs));
}

Example 23 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testRestClientStepNode.

@Test
public void testRestClientStepNode() throws Exception {
    RestClientStepNode node = root.getRestClientStepNode("REST Client");
    assertNotNull(node);
    Iterable<FramedMetaverseNode> inputUrls = node.getInputUrls();
    int countInputUrls = getIterableSize(inputUrls);
    assertEquals(1, countInputUrls);
    assertEquals("REST Client", node.getStepType());
    RestMeta stepMeta = (RestMeta) getStepMeta(node);
    for (FramedMetaverseNode inputUrl : inputUrls) {
        assertEquals(stepMeta.getUrl(), inputUrl.getName());
    }
    // check the param  field is "used"
    Iterable<StreamFieldNode> streamFieldNodesUses = node.getStreamFieldNodesUses();
    assertEquals(1, getIterableSize(streamFieldNodesUses));
    for (StreamFieldNode streamFieldNodesUse : streamFieldNodesUses) {
        assertEquals(stepMeta.getParameterField()[0], streamFieldNodesUse.getName());
    }
    Iterable<StreamFieldNode> outputs = node.getOutputStreamFields();
    assertEquals(4, getIterableSize(outputs));
    Iterable<StreamFieldNode> inputs = node.getInputStreamFields();
    assertEquals(3, getIterableSize(inputs));
    for (StreamFieldNode in : inputs) {
        assertNotNull(in.getFieldNodesDerivedFromMe());
        assertEquals(in.getName(), in.getFieldNodesDerivedFromMe().iterator().next().getName());
    }
}

Example 24 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testHTTPPostStep_UrlFromField.

@Test
public void testHTTPPostStep_UrlFromField() throws Exception {
    // this is testing a specific TextFileInputStep instance
    HttpPostStepNode node = root.getHttpPostStepNode_urlFromField();
    assertNotNull(node);
    // this HTTP Client gets it's files from an incoming stream field, there should be no files modeled statically
    Iterable<FramedMetaverseNode> inputUrls = node.getInputUrls();
    int countInputUrls = getIterableSize(inputUrls);
    assertEquals(0, countInputUrls);
    assertEquals("HTTP Post", node.getStepType());
    HTTPPOSTMeta stepMeta = (HTTPPOSTMeta) getStepMeta(node);
    Set<String> usedFields = new HashSet<>();
    Collections.addAll(usedFields, stepMeta.getQueryField());
    Collections.addAll(usedFields, stepMeta.getArgumentField());
    if (stepMeta.isUrlInField()) {
        usedFields.add(stepMeta.getUrlField());
    }
    // check the param  field is "used"
    Iterable<StreamFieldNode> streamFieldNodesUses = node.getStreamFieldNodesUses();
    assertEquals(usedFields.size(), getIterableSize(streamFieldNodesUses));
    for (StreamFieldNode streamFieldNodesUse : streamFieldNodesUses) {
        assertTrue(usedFields.contains(streamFieldNodesUse.getName()));
    }
    Iterable<StreamFieldNode> outputs = node.getOutputStreamFields();
    assertEquals(getExpectedOutputFieldCount(stepMeta), getIterableSize(outputs));
}

Example 25 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTextFileInputStep_filenameFromField.

@Test
public void testTextFileInputStep_filenameFromField() throws Exception {
    // this is testing a specific TextFileInputStep instance
    FileInputStepNode fileInputStepNode = root.getTextFileInputStepNode_filenameFromField();
    assertNotNull(fileInputStepNode);
    int countUses = getIterableSize(fileInputStepNode.getFileFieldNodesUses());
    int countInputs = getIterableSize(fileInputStepNode.getInputStreamFields());
    assertEquals(1, countUses);
    int fileFieldCount = 0;
    Iterable<StreamFieldNode> outFields = fileInputStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (!outField.getName().equals("filename")) {
            FieldNode fieldPopulatesMe = outField.getFieldPopulatesMe();
            assertNotNull(fieldPopulatesMe);
            assertEquals(DictionaryConst.NODE_TYPE_FILE_FIELD, fieldPopulatesMe.getType());
            assertEquals(fileInputStepNode, fieldPopulatesMe.getStepThatInputsMe());
            fileFieldCount++;
        }
    }
    // we should have one more input than file fields since we are reading it off of the input stream
    assertEquals(countInputs - 1, fileFieldCount);
    assertEquals(countOutputs, fileFieldCount);
    String filenameField = null;
    TransMeta tm = new TransMeta(new FileInputStream(fileInputStepNode.getTransNode().getPath()), null, true, null, null);
    for (StepMeta stepMeta : tm.getSteps()) {
        if (stepMeta.getName().equals(fileInputStepNode.getName())) {
            org.pentaho.di.trans.steps.fileinput.text.TextFileInputMeta meta = (org.pentaho.di.trans.steps.fileinput.text.TextFileInputMeta) getBaseStepMetaFromStepMeta(stepMeta);
            assertTrue(meta.isAcceptingFilenames());
            filenameField = meta.getAcceptingField();
            assertNotNull(filenameField);
            assertEquals(filenameField, fileInputStepNode.getFileFieldNodesUses().iterator().next().getName());
            // this was the one we cared about...
            break;
        }
    }
}