Examples with Operations org.pentaho.metaverse.api.model.Operations used on opensource projects

Example 6 with Operations

use of org.pentaho.metaverse.api.model.Operations in project pentaho-metaverse by pentaho.

the class MetaverseValidationIT method testSelectValuesStep.

@Test
public void testSelectValuesStep() throws Exception {
    // this tests a specific select values step. the one in trans "Populate Table From File"
    SelectValuesTransStepNode selectValues = root.getSelectValuesStepNode();
    assertNotNull(selectValues);
    int countUses = getIterableSize(selectValues.getStreamFieldNodesUses());
    int countOutputs = getIterableSize(selectValues.getOutputStreamFields());
    int countInputs = getIterableSize(selectValues.getInputStreamFields());
    assertEquals(9, countUses);
    SelectValuesMeta meta = (SelectValuesMeta) getStepMeta(selectValues);
    assertEquals(getExpectedOutputFieldCount(meta), countOutputs);
    assertEquals(9, countInputs);
    assertEquals("Select values", selectValues.getStepType());
    for (StreamFieldNode node : selectValues.getOutputStreamFields()) {
        // check for operations
        if (node.getOperations() != null) {
            Operations ops = MetaverseUtil.convertOperationsStringToMap(node.getOperations());
            assertNotNull(ops);
            List<IOperation> metadataOps = ops.get(ChangeType.METADATA);
            assertNotNull(metadataOps);
            assertTrue(metadataOps.size() > 0);
            // there should not be any data operations on nodes touched by this step
            assertNull(ops.get(ChangeType.DATA));
        }
        // check the created node is derived from something
        Iterable<StreamFieldNode> deriveNodes = node.getFieldNodesThatDeriveMe();
        for (StreamFieldNode deriveNode : deriveNodes) {
            assertNotNull(deriveNode);
        }
    }
}

Example 7 with Operations

use of org.pentaho.metaverse.api.model.Operations in project pentaho-metaverse by pentaho.

the class LineageClientIT method testGetOperationPaths.

@Test
public void testGetOperationPaths() throws Exception {
    Map<String, Set<List<StepFieldOperations>>> operationPathMap = client.getOperationPaths(transMeta, "Select values", Arrays.asList("HELLO"));
    assertNotNull(operationPathMap);
    assertEquals(1, operationPathMap.size());
    assertNull(operationPathMap.get("COUNTRY"));
    Set<List<StepFieldOperations>> operationPaths = operationPathMap.get("HELLO");
    assertNotNull(operationPaths);
    assertEquals(2, operationPaths.size());
    for (List<StepFieldOperations> operationPath : operationPaths) {
        // Should be 6 nodes along one path and 4 along the other
        int pathLength = operationPath.size();
        assertTrue(pathLength == 6 || pathLength == 4);
        // The end and last nodes should be the same for both paths
        StepFieldOperations last = operationPath.get(pathLength - 1);
        assertEquals("Select values", last.getStepName());
        assertEquals("HELLO", last.getFieldName());
        Operations ops = last.getOperations();
        assertNotNull(ops);
        assertEquals(1, ops.size());
        List<IOperation> dataOps = ops.get(ChangeType.DATA);
        assertNull(dataOps);
        List<IOperation> metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        IOperation metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        assertEquals("name", metadataOp.getDescription());
        StepFieldOperations passthru = operationPath.get(pathLength - 2);
        assertEquals("Passthru", passthru.getStepName());
        assertEquals("COUNTRY_1", passthru.getFieldName());
        metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        StepFieldOperations middle = operationPath.get(pathLength - 3);
        assertEquals("Merge Join", middle.getStepName());
        assertEquals("COUNTRY_1", middle.getFieldName());
        metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        assertEquals("name", metadataOp.getDescription());
        StepFieldOperations first = operationPath.get(0);
        assertEquals("COUNTRY", first.getFieldName());
        // The step name is either "Table input" or "Data Grid"
        String firstStepName = first.getStepName();
        assertTrue("Table input".equals(firstStepName) || "Data Grid".equals(firstStepName));
        if (pathLength == 4) {
            assertEquals("COUNTRY", first.getFieldName());
            // The step name is either "Table input" or "Data Grid"
            firstStepName = first.getStepName();
            assertTrue("Table input".equals(firstStepName) || "Data Grid".equals(firstStepName));
        }
    }
}

Example 8 with Operations

use of org.pentaho.metaverse.api.model.Operations in project pentaho-metaverse by pentaho.

the class MetaverseUtil method convertOperationsStringToMap.

public static Operations convertOperationsStringToMap(String operations) {
    Operations resultOps = null;
    if (!Const.isEmpty(operations)) {
        try {
            Map<String, List<IOperation>> rawOpsMap = new JSONDeserializer<Map<String, List<IOperation>>>().use("values.values", Operation.class).deserialize(operations);
            resultOps = new Operations();
            for (String key : rawOpsMap.keySet()) {
                resultOps.put(ChangeType.forValue(key), rawOpsMap.get(key));
            }
        } catch (Exception e) {
            resultOps = null;
        }
    // return new JSONDeserializer<Operations>().use(null, Operations.class).deserialize( operations );
    }
    return resultOps;
}

Example 9 with Operations

use of org.pentaho.metaverse.api.model.Operations in project pentaho-metaverse by pentaho.

the class SplitFieldsStepAnalyzerTest method testGetChangeRecords.

@Test
public void testGetChangeRecords() throws Exception {
    Set<ComponentDerivationRecord> changeRecords = analyzer.getChangeRecords(meta);
    assertEquals(3, changeRecords.size());
    for (ComponentDerivationRecord changeRecord : changeRecords) {
        assertEquals("splitField", changeRecord.getOriginalEntityName());
        assertTrue(ArrayUtils.contains(outputFields, changeRecord.getChangedEntityName()));
        Operations operations = changeRecord.getOperations();
        // Only data operations
        assertEquals(1, operations.size());
        List<IOperation> dataOperations = operations.get(ChangeType.DATA);
        assertEquals(1, dataOperations.size());
    }
}

Example 10 with Operations

use of org.pentaho.metaverse.api.model.Operations in project pentaho-metaverse by pentaho.

the class ValueMapperStepAnalyzerTest method testGetChangeRecords.

@Test
public void testGetChangeRecords() throws Exception {
    Set<ComponentDerivationRecord> changeRecords = analyzer.getChangeRecords(meta);
    assertEquals(1, changeRecords.size());
    ComponentDerivationRecord changeRecord = changeRecords.iterator().next();
    assertEquals(meta.getFieldToUse(), changeRecord.getOriginalEntityName());
    assertEquals((meta.getTargetField() == null ? meta.getFieldToUse() : meta.getTargetField()), changeRecord.getChangedEntityName());
    Operations operations = changeRecord.getOperations();
    // Only data operations
    assertEquals(1, operations.size());
    List<IOperation> dataOperations = operations.get(ChangeType.DATA);
    assertEquals(2, dataOperations.size());
}