Examples with IOperation org.pentaho.metaverse.api.model.IOperation used on opensource projects

Example 1 with IOperation

use of org.pentaho.metaverse.api.model.IOperation in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testNumberRangeStepNode_newField.

@Test
public void testNumberRangeStepNode_newField() throws Exception {
    TransformationStepNode numberRangeStepNode = root.getStepNode("number_range", "Number range");
    assertEquals(1, getIterableSize(numberRangeStepNode.getStreamFieldNodesUses()));
    StreamFieldNode usesNode = null;
    for (StreamFieldNode node : numberRangeStepNode.getStreamFieldNodesUses()) {
        usesNode = node;
        break;
    }
    NumberRangeMeta meta = (NumberRangeMeta) getStepMeta(numberRangeStepNode);
    assertEquals(meta.getInputField(), usesNode.getName());
    assertNull(usesNode.getOperations());
    Iterable<StreamFieldNode> outFields = numberRangeStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    assertEquals(2, countOutputs);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (outField.getName().equals(meta.getOutputField())) {
            Iterable<StreamFieldNode> fieldNodesThatDeriveMe = outField.getFieldNodesThatDeriveMe();
            assertNotNull(fieldNodesThatDeriveMe);
            int derivesCount = getIterableSize(outField.getFieldNodesThatDeriveMe());
            assertEquals(1, derivesCount);
            for (FieldNode derives : outField.getFieldNodesThatDeriveMe()) {
                assertEquals(usesNode.getName(), derives.getName());
                assertEquals(usesNode.getType(), derives.getType());
            }
            Operations ops = MetaverseUtil.convertOperationsStringToMap(outField.getOperations());
            List<IOperation> dataOps = ops.get(ChangeType.DATA);
            assertNotNull(dataOps);
            assertEquals(3, dataOps.size());
            for (IOperation dataOp : dataOps) {
                assertEquals(Operation.MAPPING_CATEGORY, dataOp.getCategory());
                assertEquals(DictionaryConst.PROPERTY_TRANSFORMS, dataOp.getName());
                assertTrue(dataOp.toString().contains(meta.getInputField()));
            }
            // there should not be any metadata operations
            assertNull(ops.get(ChangeType.METADATA));
        }
    }
}

Example 2 with IOperation

use of org.pentaho.metaverse.api.model.IOperation in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testValueMapperStepNode_newField.

@Test
public void testValueMapperStepNode_newField() throws Exception {
    TransformationStepNode valueMapperStepNode = root.getStepNode("value_mapper", "Value Mapper - new field");
    assertEquals(1, getIterableSize(valueMapperStepNode.getStreamFieldNodesUses()));
    StreamFieldNode usesNode = valueMapperStepNode.getStreamFieldNodesUses().iterator().next();
    ValueMapperMeta meta = (ValueMapperMeta) getStepMeta(valueMapperStepNode);
    assertEquals(meta.getFieldToUse(), usesNode.getName());
    assertNull(usesNode.getOperations());
    Iterable<StreamFieldNode> inFields = valueMapperStepNode.getInputStreamFields();
    int countInputs = getIterableSize(inFields);
    Iterable<StreamFieldNode> outFields = valueMapperStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    assertEquals(countInputs + 1, countOutputs);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (outField.getName().equals(meta.getTargetField())) {
            Iterable<StreamFieldNode> fieldNodesThatDeriveMe = outField.getFieldNodesThatDeriveMe();
            assertNotNull(fieldNodesThatDeriveMe);
            int derivesCount = getIterableSize(outField.getFieldNodesThatDeriveMe());
            assertEquals(1, derivesCount);
            for (FieldNode derives : outField.getFieldNodesThatDeriveMe()) {
                assertEquals(usesNode.getName(), derives.getName());
                assertEquals(usesNode.getType(), derives.getType());
            }
            Operations ops = MetaverseUtil.convertOperationsStringToMap(outField.getOperations());
            List<IOperation> dataOps = ops.get(ChangeType.DATA);
            assertNotNull(dataOps);
            assertEquals(1, dataOps.size());
            for (IOperation dataOp : dataOps) {
                assertEquals(Operation.MAPPING_CATEGORY, dataOp.getCategory());
                assertEquals(DictionaryConst.PROPERTY_TRANSFORMS, dataOp.getName());
            }
            // there should not be any metadata operations
            assertNull(ops.get(ChangeType.METADATA));
        }
    }
}

Example 3 with IOperation

use of org.pentaho.metaverse.api.model.IOperation in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testValueMapperStepNode_overwrite.

@Test
public void testValueMapperStepNode_overwrite() throws Exception {
    TransformationStepNode valueMapperStepNode = root.getStepNode("value_mapper", "Value Mapper - overwrite");
    assertEquals(1, getIterableSize(valueMapperStepNode.getStreamFieldNodesUses()));
    StreamFieldNode usesNode = valueMapperStepNode.getStreamFieldNodesUses().iterator().next();
    ValueMapperMeta meta = (ValueMapperMeta) getStepMeta(valueMapperStepNode);
    assertEquals(meta.getFieldToUse(), usesNode.getName());
    Iterable<StreamFieldNode> inFields = valueMapperStepNode.getInputStreamFields();
    int countInputs = getIterableSize(inFields);
    Iterable<StreamFieldNode> outFields = valueMapperStepNode.getOutputStreamFields();
    int countOutputs = getIterableSize(outFields);
    assertEquals(countInputs, countOutputs);
    for (StreamFieldNode outField : outFields) {
        assertNotNull(outField.getKettleType());
        if (outField.getName().equals(meta.getTargetField())) {
            Operations ops = MetaverseUtil.convertOperationsStringToMap(usesNode.getOperations());
            List<IOperation> dataOps = ops.get(ChangeType.DATA);
            assertNotNull(dataOps);
            assertEquals(meta.getSourceValue().length, dataOps.size());
            for (int i = 0; i < dataOps.size(); i++) {
                IOperation dataOp = dataOps.get(i);
                assertEquals(DictionaryConst.PROPERTY_TRANSFORMS, dataOp.getName());
                assertEquals(dataOp.toString(), meta.getSourceValue()[i] + " -> " + meta.getTargetValue()[i]);
            }
            // there should not be any metadata operations
            assertNull(ops.get(ChangeType.METADATA));
        }
    }
    int derivedCount = getIterableSize(usesNode.getFieldNodesDerivedFromMe());
    assertEquals(1, derivedCount);
}

Example 4 with IOperation

use of org.pentaho.metaverse.api.model.IOperation in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testSelectValuesStep.

@Test
public void testSelectValuesStep() throws Exception {
    // this tests a specific select values step. the one in trans "Populate Table From File"
    SelectValuesTransStepNode selectValues = root.getSelectValuesStepNode();
    assertNotNull(selectValues);
    int countUses = getIterableSize(selectValues.getStreamFieldNodesUses());
    int countOutputs = getIterableSize(selectValues.getOutputStreamFields());
    int countInputs = getIterableSize(selectValues.getInputStreamFields());
    assertEquals(9, countUses);
    SelectValuesMeta meta = (SelectValuesMeta) getStepMeta(selectValues);
    assertEquals(getExpectedOutputFieldCount(meta), countOutputs);
    assertEquals(9, countInputs);
    assertEquals("Select values", selectValues.getStepType());
    for (StreamFieldNode node : selectValues.getOutputStreamFields()) {
        // check for operations
        if (node.getOperations() != null) {
            Operations ops = MetaverseUtil.convertOperationsStringToMap(node.getOperations());
            assertNotNull(ops);
            List<IOperation> metadataOps = ops.get(ChangeType.METADATA);
            assertNotNull(metadataOps);
            assertTrue(metadataOps.size() > 0);
            // there should not be any data operations on nodes touched by this step
            assertNull(ops.get(ChangeType.DATA));
        }
        // check the created node is derived from something
        Iterable<StreamFieldNode> deriveNodes = node.getFieldNodesThatDeriveMe();
        for (StreamFieldNode deriveNode : deriveNodes) {
            assertNotNull(deriveNode);
        }
    }
}

Example 5 with IOperation

use of org.pentaho.metaverse.api.model.IOperation in project pentaho-metaverse by pentaho.

the class LineageClientIT method testGetOperationPaths.

@Test
public void testGetOperationPaths() throws Exception {
    Map<String, Set<List<StepFieldOperations>>> operationPathMap = client.getOperationPaths(transMeta, "Select values", Arrays.asList("HELLO"));
    assertNotNull(operationPathMap);
    assertEquals(1, operationPathMap.size());
    assertNull(operationPathMap.get("COUNTRY"));
    Set<List<StepFieldOperations>> operationPaths = operationPathMap.get("HELLO");
    assertNotNull(operationPaths);
    assertEquals(2, operationPaths.size());
    for (List<StepFieldOperations> operationPath : operationPaths) {
        // Should be 6 nodes along one path and 4 along the other
        int pathLength = operationPath.size();
        assertTrue(pathLength == 6 || pathLength == 4);
        // The end and last nodes should be the same for both paths
        StepFieldOperations last = operationPath.get(pathLength - 1);
        assertEquals("Select values", last.getStepName());
        assertEquals("HELLO", last.getFieldName());
        Operations ops = last.getOperations();
        assertNotNull(ops);
        assertEquals(1, ops.size());
        List<IOperation> dataOps = ops.get(ChangeType.DATA);
        assertNull(dataOps);
        List<IOperation> metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        IOperation metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        assertEquals("name", metadataOp.getDescription());
        StepFieldOperations passthru = operationPath.get(pathLength - 2);
        assertEquals("Passthru", passthru.getStepName());
        assertEquals("COUNTRY_1", passthru.getFieldName());
        metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        StepFieldOperations middle = operationPath.get(pathLength - 3);
        assertEquals("Merge Join", middle.getStepName());
        assertEquals("COUNTRY_1", middle.getFieldName());
        metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        assertEquals("name", metadataOp.getDescription());
        StepFieldOperations first = operationPath.get(0);
        assertEquals("COUNTRY", first.getFieldName());
        // The step name is either "Table input" or "Data Grid"
        String firstStepName = first.getStepName();
        assertTrue("Table input".equals(firstStepName) || "Data Grid".equals(firstStepName));
        if (pathLength == 4) {
            assertEquals("COUNTRY", first.getFieldName());
            // The step name is either "Table input" or "Data Grid"
            firstStepName = first.getStepName();
            assertTrue("Table input".equals(firstStepName) || "Data Grid".equals(firstStepName));
        }
    }
}