Examples with StreamFieldNode org.pentaho.metaverse.frames.StreamFieldNode used on opensource projects

Example 16 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testStreamLookupStepNode.

@Test
public void testStreamLookupStepNode() throws Exception {
    StreamLookupStepNode node = root.getStreamLookupStepNode();
    assertEquals(6, getIterableSize(node.getInputStreamFields()));
    assertEquals(3, getIterableSize(node.getStreamFieldNodesUses()));
    assertEquals(5, getIterableSize(node.getOutputStreamFields()));
    List<String> expectations = new ArrayList<>();
    expectations.add("territory");
    expectations.add("country_ref");
    Iterator<StreamFieldNode> iter1 = node.getOutputStreamFields().iterator();
    while (iter1.hasNext()) {
        StreamFieldNode outField = iter1.next();
        if (expectations.contains(outField.getName())) {
            assertEquals(1, getIterableSize(outField.getFieldNodesThatDeriveMe()));
            Iterator<StreamFieldNode> iter2 = outField.getFieldNodesThatDeriveMe().iterator();
            while (iter2.hasNext()) {
                StreamFieldNode derivedFromNode = iter2.next();
                assertTrue(expectations.contains(derivedFromNode.getName()));
            }
        }
    }
    iter1 = node.getStreamFieldNodesUses().iterator();
    while (iter1.hasNext()) {
        StreamFieldNode usesNode = iter1.next();
        if (usesNode.getName().equals("country_code") || usesNode.getName().equals("country_code")) {
            assertEquals(1, getIterableSize(usesNode.getFieldNodesThatJoinToMe()));
            StreamFieldNode joinField = usesNode.getFieldNodesThatJoinToMe().iterator().next();
            if (usesNode.getName().equals("country_code")) {
                assertEquals("code", joinField.getName());
            } else {
                assertEquals("country_code", joinField.getName());
            }
        }
    }
}

Example 17 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testFilterRowsStepNode.

@Test
public void testFilterRowsStepNode() throws Exception {
    FilterRowsStepNode node = root.getFilterRowsStepNode("Filter rows");
    assertNotNull(node);
    assertEquals("Filter rows", node.getStepType());
    FilterRowsMeta meta = (FilterRowsMeta) getStepMeta(node);
    Operations ops = MetaverseUtil.convertOperationsStringToMap(node.getOperations());
    assertEquals(1, ops.get(ChangeType.DATA_FLOW).size());
    assertEquals(meta.getCondition().toString(), ops.get(ChangeType.DATA_FLOW).get(0).getDescription());
    // should not be any created nodes
    Iterable<StreamFieldNode> streamFieldNodes = node.getStreamFieldNodesCreates();
    int countCreatedStreamFieldNode = getIterableSize(streamFieldNodes);
    assertEquals(0, countCreatedStreamFieldNode);
    // should not be any deleted nodes
    streamFieldNodes = node.getStreamFieldNodesCreates();
    int countDeletedStreamFieldNode = getIterableSize(streamFieldNodes);
    assertEquals(0, countDeletedStreamFieldNode);
    // should use all of the fields that are used in the condition of the step
    List<String> expectedUses = Arrays.asList(meta.getCondition().getUsedFields());
    streamFieldNodes = node.getStreamFieldNodesUses();
    int countUsedStreamFieldNode = getIterableSize(streamFieldNodes);
    assertEquals(expectedUses.size(), countUsedStreamFieldNode);
    for (StreamFieldNode streamFieldNode : streamFieldNodes) {
        assertTrue(expectedUses.contains(streamFieldNode.getName()));
    }
}

Example 18 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testSelectValuesStep.

@Test
public void testSelectValuesStep() throws Exception {
    // this tests a specific select values step. the one in trans "Populate Table From File"
    SelectValuesTransStepNode selectValues = root.getSelectValuesStepNode();
    assertNotNull(selectValues);
    int countUses = getIterableSize(selectValues.getStreamFieldNodesUses());
    int countOutputs = getIterableSize(selectValues.getOutputStreamFields());
    int countInputs = getIterableSize(selectValues.getInputStreamFields());
    assertEquals(9, countUses);
    SelectValuesMeta meta = (SelectValuesMeta) getStepMeta(selectValues);
    assertEquals(getExpectedOutputFieldCount(meta), countOutputs);
    assertEquals(9, countInputs);
    assertEquals("Select values", selectValues.getStepType());
    for (StreamFieldNode node : selectValues.getOutputStreamFields()) {
        // check for operations
        if (node.getOperations() != null) {
            Operations ops = MetaverseUtil.convertOperationsStringToMap(node.getOperations());
            assertNotNull(ops);
            List<IOperation> metadataOps = ops.get(ChangeType.METADATA);
            assertNotNull(metadataOps);
            assertTrue(metadataOps.size() > 0);
            // there should not be any data operations on nodes touched by this step
            assertNull(ops.get(ChangeType.DATA));
        }
        // check the created node is derived from something
        Iterable<StreamFieldNode> deriveNodes = node.getFieldNodesThatDeriveMe();
        for (StreamFieldNode deriveNode : deriveNodes) {
            assertNotNull(deriveNode);
        }
    }
}

Example 19 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testTableOutputStepNode.

@Test
public void testTableOutputStepNode() throws Exception {
    // this tests a specific step in a specific transform
    TableOutputStepNode node = root.getTableOutputStepNode();
    // check the table that it writes to
    TableOutputMeta meta = (TableOutputMeta) getStepMeta(node);
    String tableName = meta.getTableName();
    String schema = meta.getSchemaName();
    boolean truncateTable = meta.truncateTable();
    DatabaseTableNode databaseTableNode = node.getDatabaseTable();
    assertEquals(tableName, databaseTableNode.getName());
    assertEquals(schema, node.getSchema());
    assertEquals(truncateTable, node.isTruncateTable());
    Iterable<StreamFieldNode> inputs = node.getInputStreamFields();
    Iterable<StreamFieldNode> outputs = node.getOutputStreamFields();
    assertEquals(getIterableSize(inputs) + meta.getFieldDatabase().length, getIterableSize(outputs));
    for (StreamFieldNode input : inputs) {
        assertEquals(input.getName(), input.getFieldPopulatedByMe().getName());
    }
    DatasourceNode datasource = node.getDatasource(meta.getDatabaseMeta().getName());
    assertEquals(meta.getDatabaseMeta().getHostname(), datasource.getHost());
    assertEquals(meta.getDatabaseMeta().getDatabasePortNumberString(), datasource.getPort());
    assertEquals(meta.getDatabaseMeta().getUsername(), datasource.getUserName());
    assertEquals(meta.getDatabaseMeta().getDatabaseName(), datasource.getDatabaseName());
    assertEquals(DictionaryConst.NODE_TYPE_DATASOURCE, datasource.getType());
    assertEquals(meta.getTableName(), databaseTableNode.getName());
    assertEquals(DictionaryConst.NODE_TYPE_DATA_TABLE, databaseTableNode.getType());
}

Example 20 with StreamFieldNode

use of org.pentaho.metaverse.frames.StreamFieldNode in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testSplitFieldsStepNode_reuseTheSplitFieldNameInOutputField.

@Test
public void testSplitFieldsStepNode_reuseTheSplitFieldNameInOutputField() throws Exception {
    SplitFieldsStepNode node = root.getSplitFieldsStepNodeByName("Split Fields - team");
    assertNotNull(node);
    FieldSplitterMeta meta = (FieldSplitterMeta) getStepMeta(node);
    assertEquals(meta.getDelimiter(), node.getDelimiter());
    assertEquals(meta.getEnclosure(), node.getEnclosure());
    Iterable<StreamFieldNode> usedFields = node.getStreamFieldNodesUses();
    int usedFieldCount = getIterableSize(usedFields);
    // make sure the node that was used is the split field
    assertEquals(meta.getSplitField(), usedFields.iterator().next().getName());
    assertEquals(1, usedFieldCount);
    // make sure the split field derives all of the output fields
    // make sure the split field derives all of the output fields
    Iterable<StreamFieldNode> outFields = node.getOutputStreamFields();
    for (StreamFieldNode outField : outFields) {
        Iterable<StreamFieldNode> derivingNodes = outField.getFieldNodesThatDeriveMe();
        int derivesFieldCount = getIterableSize(derivingNodes);
        assertEquals(1, derivesFieldCount);
        StreamFieldNode derivingField = derivingNodes.iterator().next();
        if (!derivingField.getName().equals("position")) {
            assertEquals(derivingField.getName(), meta.getSplitField());
        }
    }
}