Examples with StreamFieldNode org.pentaho.metaverse.frames.StreamFieldNode used on opensource projects

Example 26 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTextFileInputStep.

@Test
public void testTextFileInputStep() throws Exception {
    // this is testing a specific TextFileInputStep instance
    FileInputStepNode fileInputStepNode = (FileInputStepNode) root.getFileInputStepNode("Populate Table From File", "Sacramento crime stats 2006 file");
    assertNotNull(fileInputStepNode);
    Iterable<FramedMetaverseNode> inputFiles = fileInputStepNode.getInputFiles();
    int countInputFiles = getIterableSize(inputFiles);
    assertEquals(1, countInputFiles);
    for (FramedMetaverseNode inputFile : inputFiles) {
        assertTrue(inputFile.getName().endsWith("SacramentocrimeJanuary2006.csv"));
    }
    assertEquals("Old Text file input", fileInputStepNode.getStepType());
    int countUses = getIterableSize(fileInputStepNode.getFileFieldNodesUses());
    int countInputs = getIterableSize(fileInputStepNode.getInputStreamFields());
    assertEquals(0, countUses);
    int fileFieldCount = 0;
    Iterable<StreamFieldNode> outFields = fileInputStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        FieldNode fieldPopulatesMe = outField.getFieldPopulatesMe();
        assertNotNull(fieldPopulatesMe);
        assertEquals(DictionaryConst.NODE_TYPE_FILE_FIELD, fieldPopulatesMe.getType());
        assertEquals(fileInputStepNode, fieldPopulatesMe.getStepThatInputsMe());
        fileFieldCount++;
    }
    assertEquals(countInputs, fileFieldCount);
    assertEquals(countOutputs, fileFieldCount);
}

Example 27 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testRowsToResultStepNode.

@Test
public void testRowsToResultStepNode() throws Exception {
    RowsToResultStepNode rowsToResultStepNode = root.getRowsToResultStepNode();
    assertNotNull(rowsToResultStepNode);
    assertEquals("Copy rows to result", rowsToResultStepNode.getStepType());
    Iterable<StreamFieldNode> streamFieldNodes = rowsToResultStepNode.getStreamFieldNodesCreates();
    int countCreatedStreamFieldNode = getIterableSize(streamFieldNodes);
    for (StreamFieldNode streamFieldNode : streamFieldNodes) {
        assertNotNull(streamFieldNode.getKettleType());
        // each field created is derived from an incoming field named the same thing
        assertEquals(streamFieldNode.getName(), streamFieldNode.getFieldNodesThatDeriveMe().iterator().next().getName());
    }
    streamFieldNodes = rowsToResultStepNode.getStreamFieldNodesUses();
    int countUsedStreamFieldNode = getIterableSize(streamFieldNodes);
    for (StreamFieldNode streamFieldNode : streamFieldNodes) {
        assertNotNull(streamFieldNode.getKettleType());
    }
    // should create a field for each one it uses
    assertEquals(countCreatedStreamFieldNode, countUsedStreamFieldNode);
}

Example 28 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testExcelOutputStepNode.

@Test
public void testExcelOutputStepNode() throws Exception {
    ExcelOutputStepNode excelOutputStepNode = root.getExcelOutputStepNode();
    assertNotNull(excelOutputStepNode);
    ExcelOutputMeta meta = (ExcelOutputMeta) getStepMeta(excelOutputStepNode);
    TransMeta tm = meta.getParentStepMeta().getParentTransMeta();
    String[] fileNames = meta.getFiles(tm);
    RowMetaInterface incomingFields = tm.getStepFields(meta.getParentStepMeta());
    ExcelField[] outputFields = meta.getOutputFields();
    // should write to one file
    Iterable<FramedMetaverseNode> outputFiles = excelOutputStepNode.getOutputFiles();
    assertEquals(fileNames.length, getIterableSize(outputFiles));
    int i = 0;
    for (FramedMetaverseNode node : outputFiles) {
        assertEquals(normalizeFilePath(fileNames[i++]), normalizeFilePath(node.getName()));
    }
    Iterable<StreamFieldNode> outFields = excelOutputStepNode.getOutputStreamFields();
    int outFieldCount = getIterableSize(outFields);
    // should have output stream nodes as well as file nodes
    assertEquals(outputFields.length * 2, outFieldCount);
    int fileFieldCount = 0;
    for (StreamFieldNode outField : outFields) {
        if (DictionaryConst.NODE_TYPE_FILE_FIELD.equals(outField.getType())) {
            ValueMetaInterface vmi = incomingFields.searchValueMeta(outField.getName());
            assertEquals(vmi.getName(), outField.getFieldPopulatesMe().getName());
            fileFieldCount++;
        }
    }
    assertEquals(fileFieldCount, outFieldCount / 2);
}

Example 29 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testHTTPClientStep_UrlFromField.

@Test
public void testHTTPClientStep_UrlFromField() throws Exception {
    // this is testing a specific TextFileInputStep instance
    HttpClientStepNode node = root.getHttpClientStepNode_urlFromField();
    assertNotNull(node);
    // this HTTP Client gets it's files from an incoming stream field, there should be no files modeled statically
    Iterable<FramedMetaverseNode> inputUrls = node.getInputUrls();
    int countInputUrls = getIterableSize(inputUrls);
    // it is coming from a field
    assertEquals(0, countInputUrls);
    HTTPMeta stepMeta = (HTTPMeta) getStepMeta(node);
    assertEquals("HTTP Client", node.getStepType());
    Set<String> usedFields = new HashSet<>();
    Collections.addAll(usedFields, stepMeta.getHeaderField());
    Collections.addAll(usedFields, stepMeta.getArgumentField());
    if (stepMeta.isUrlInField()) {
        usedFields.add(stepMeta.getUrlField());
    }
    // check the param  field is "used"
    Iterable<StreamFieldNode> streamFieldNodesUses = node.getStreamFieldNodesUses();
    assertEquals(usedFields.size(), getIterableSize(streamFieldNodesUses));
    for (StreamFieldNode streamFieldNodesUse : streamFieldNodesUses) {
        assertTrue(usedFields.contains(streamFieldNodesUse.getName()));
    }
    Iterable<StreamFieldNode> outputs = node.getOutputStreamFields();
    assertEquals(getExpectedOutputFieldCount(stepMeta), getIterableSize(outputs));
}

Example 30 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTextFileOutputStepNode_FileFromStreamField.

@Test
public void testTextFileOutputStepNode_FileFromStreamField() throws Exception {
    TextFileOutputStepNode textFileOutputStepNode = root.getTextFileOutputStepNode("textFileOutput", "Text file output - file from field");
    TextFileOutputMeta meta = (TextFileOutputMeta) getStepMeta(textFileOutputStepNode);
    TransMeta tm = meta.getParentStepMeta().getParentTransMeta();
    RowMetaInterface incomingFields = tm.getPrevStepFields(meta.getParentStepMeta());
    int outputFields = getExpectedOutputFieldCount(meta);
    assertNotNull(textFileOutputStepNode);
    // should write to one file
    Iterable<FramedMetaverseNode> outputFiles = textFileOutputStepNode.getOutputFiles();
    assertEquals(0, getIterableSize(outputFiles));
    Iterable<StreamFieldNode> usedFields = textFileOutputStepNode.getStreamFieldNodesUses();
    int usedFieldCount = getIterableSize(usedFields);
    assertEquals(1, usedFieldCount);
    Iterable<StreamFieldNode> outFields = textFileOutputStepNode.getOutputStreamFields();
    int outFieldCount = getIterableSize(outFields);
    // should have output stream nodes as well as file nodes
    assertEquals(outputFields + meta.getOutputFields().length, outFieldCount);
    int fileFieldCount = 0;
    for (StreamFieldNode outField : outFields) {
        if (DictionaryConst.NODE_TYPE_FILE_FIELD.equals(outField.getType())) {
            ValueMetaInterface vmi = incomingFields.searchValueMeta(outField.getName());
            assertEquals(vmi.getName(), outField.getFieldPopulatesMe().getName());
            fileFieldCount++;
        }
    }
    assertEquals(fileFieldCount, outFieldCount / 2);
}