Examples with StepFieldOperations org.pentaho.metaverse.api.StepFieldOperations used on opensource projects

Example 1 with StepFieldOperations

use of org.pentaho.metaverse.api.StepFieldOperations in project pentaho-metaverse by pentaho.

the class LineageClientIT method testGetOperationPaths.

@Test
public void testGetOperationPaths() throws Exception {
    Map<String, Set<List<StepFieldOperations>>> operationPathMap = client.getOperationPaths(transMeta, "Select values", Arrays.asList("HELLO"));
    assertNotNull(operationPathMap);
    assertEquals(1, operationPathMap.size());
    assertNull(operationPathMap.get("COUNTRY"));
    Set<List<StepFieldOperations>> operationPaths = operationPathMap.get("HELLO");
    assertNotNull(operationPaths);
    assertEquals(2, operationPaths.size());
    for (List<StepFieldOperations> operationPath : operationPaths) {
        // Should be 6 nodes along one path and 4 along the other
        int pathLength = operationPath.size();
        assertTrue(pathLength == 6 || pathLength == 4);
        // The end and last nodes should be the same for both paths
        StepFieldOperations last = operationPath.get(pathLength - 1);
        assertEquals("Select values", last.getStepName());
        assertEquals("HELLO", last.getFieldName());
        Operations ops = last.getOperations();
        assertNotNull(ops);
        assertEquals(1, ops.size());
        List<IOperation> dataOps = ops.get(ChangeType.DATA);
        assertNull(dataOps);
        List<IOperation> metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        IOperation metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        assertEquals("name", metadataOp.getDescription());
        StepFieldOperations passthru = operationPath.get(pathLength - 2);
        assertEquals("Passthru", passthru.getStepName());
        assertEquals("COUNTRY_1", passthru.getFieldName());
        metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        StepFieldOperations middle = operationPath.get(pathLength - 3);
        assertEquals("Merge Join", middle.getStepName());
        assertEquals("COUNTRY_1", middle.getFieldName());
        metadataOps = ops.get(ChangeType.METADATA);
        assertNotNull(metadataOps);
        assertEquals(1, metadataOps.size());
        metadataOp = metadataOps.get(0);
        assertEquals(IOperation.METADATA_CATEGORY, metadataOp.getCategory());
        assertEquals(DictionaryConst.PROPERTY_MODIFIED, metadataOp.getName());
        assertEquals("name", metadataOp.getDescription());
        StepFieldOperations first = operationPath.get(0);
        assertEquals("COUNTRY", first.getFieldName());
        // The step name is either "Table input" or "Data Grid"
        String firstStepName = first.getStepName();
        assertTrue("Table input".equals(firstStepName) || "Data Grid".equals(firstStepName));
        if (pathLength == 4) {
            assertEquals("COUNTRY", first.getFieldName());
            // The step name is either "Table input" or "Data Grid"
            firstStepName = first.getStepName();
            assertTrue("Table input".equals(firstStepName) || "Data Grid".equals(firstStepName));
        }
    }
}

Example 2 with StepFieldOperations

use of org.pentaho.metaverse.api.StepFieldOperations in project pdi-dataservice-server-plugin by pentaho.

the class AutoParameterGenerationServiceTest method testApplyWithExisting.

@Test
public void testApplyWithExisting() throws Exception {
    Map<String, Set<List<StepFieldOperations>>> operationPaths;
    List<StepFieldOperations> inputOperations = ImmutableList.of(new StepFieldOperations("Input", "field", null));
    operationPaths = ImmutableMap.of("Input 1", Collections.singleton(inputOperations));
    when(lineageClient.getOperationPaths(same(transMeta), eq(SERVICE_STEP), eq(ImmutableList.copyOf(SERVICE_FIELDS)))).thenReturn(operationPaths);
    StepMeta input1Meta = mock(StepMeta.class);
    when(transMeta.findStep("Input")).thenReturn(input1Meta);
    when(serviceProvider.supportsStep(input1Meta)).thenReturn(true);
    assertThat(service.parametrizedSteps(dataService), empty());
    assertThat(service.apply(dataService), hasSize(1));
    PushDownOptimizationMeta optimizationMeta = new PushDownOptimizationMeta();
    optimizationMeta.setName("Existing PDO");
    optimizationMeta.setStepName("Input");
    optimizationMeta.setType(mock(ParameterGeneration.class));
    dataService.setPushDownOptimizationMeta(Lists.newArrayList(optimizationMeta));
    assertThat(service.parametrizedSteps(dataService), contains("Input"));
    assertThat(service.apply(dataService), empty());
}

Example 3 with StepFieldOperations

use of org.pentaho.metaverse.api.StepFieldOperations in project pentaho-metaverse by pentaho.

the class LineageClient method getOperationPaths.

/**
 * Returns the paths between the origin field(s) and target field(s). A path in this context is an ordered list of
 * StepFieldOperations objects, each of which corresponds to a field at a certain step where operation(s) are
 * applied. The order of the list corresponds to the order of the steps from the origin step (see getOriginSteps())
 * to the target step. This method can be used to trace a target field back to its origin and discovering what
 * operations were performed upon it during it's lifetime. Inversely the path could be used to re-apply the operations
 * to the origin field, resulting in the field's "value" at each point in the path.
 *
 * @param transMeta      a reference to a transformation's metadata
 * @param targetStepName the target step name associated with the given field names
 * @param fieldNames     an array of field names associated with the target step, for which to find the step(s) and
 *                       field(s) and operation(s) that contributed to those fields
 * @return a map of target field name to an ordered list of StepFieldOperations objects, describing the path from the
 * origin step field to the target step field, including the operations performed.
 * @throws MetaverseException if an error occurred while finding the origin steps
 */
@Override
public Map<String, Set<List<StepFieldOperations>>> getOperationPaths(TransMeta transMeta, String targetStepName, final Collection<String> fieldNames) throws MetaverseException {
    Map<String, Set<List<StepFieldOperations>>> operationPathMap = new HashMap<>();
    try {
        Future<Graph> lineageGraphTask = LineageGraphMap.getInstance().get(transMeta);
        if (lineageGraphTask != null) {
            Graph lineageGraph = lineageGraphTask.get();
            if (lineageGraph != null) {
                // Get the creator field nodes for all the field names passed in
                List<Vertex> getTargetFields = getTargetFields(lineageGraph, targetStepName, fieldNames);
                // The "origin steps pipe" with a second param of true returns a pipeline that will return paths between
                // the origin field nodes and the target field node.
                GremlinPipeline pipe = getOriginStepsPipe(getTargetFields);
                List<List<Vertex>> pathList = pipe.toList();
                if (pathList != null) {
                    for (List<Vertex> path : pathList) {
                        // Transform each path of vertices into a "path" of StepFieldOperations objects (basically save off
                        // properties of each vertex into a new list)
                        List<StepFieldOperations> stepFieldOps = new ArrayList<>();
                        String targetField = path.get(0).getProperty(DictionaryConst.PROPERTY_NAME);
                        Set<List<StepFieldOperations>> pathSet = operationPathMap.get(targetField);
                        if (pathSet == null) {
                            pathSet = new HashSet<>();
                            operationPathMap.put(targetField, pathSet);
                        }
                        for (Vertex v : path) {
                            Map<String, String> stepField = STEPFIELDOPS_PIPE_FUNC.compute(v);
                            String stepName = stepField.get("stepName");
                            String fieldName = stepField.get("fieldName");
                            Operations operations = MetaverseUtil.convertOperationsStringToMap((String) v.getProperty(DictionaryConst.PROPERTY_OPERATIONS));
                            stepFieldOps.add(0, new StepFieldOperations(stepName, fieldName, operations));
                        }
                        pathSet.add(stepFieldOps);
                    }
                }
            }
        }
    } catch (Exception e) {
        throw new MetaverseException(e);
    }
    return operationPathMap;
}

Example 4 with StepFieldOperations

use of org.pentaho.metaverse.api.StepFieldOperations in project pdi-dataservice-server-plugin by pentaho.

the class AutoParameterGenerationServiceTest method testApply.

@Test
public void testApply() throws Exception {
    Map<String, Set<List<StepFieldOperations>>> operationPaths;
    List<StepFieldOperations> field1_input1 = Lists.newArrayList(new StepFieldOperations("Input 1", "field1_origin", null), new StepFieldOperations("Modifier", "field1", new MockOperations().put(ChangeType.DATA).put(ChangeType.METADATA)));
    List<StepFieldOperations> field2_input1 = Lists.newArrayList(new StepFieldOperations("Input 1", "field2_origin", null), new StepFieldOperations("Rename", "field2", new MockOperations().put(ChangeType.METADATA)));
    List<StepFieldOperations> field2_input2 = Lists.newArrayList(new StepFieldOperations("Input 2", "field2", null));
    operationPaths = Maps.newHashMap();
    operationPaths.put("field1", ImmutableSet.of(field1_input1));
    operationPaths.put("field2", ImmutableSet.of(field2_input1, field2_input2));
    when(lineageClient.getOperationPaths(same(transMeta), eq(SERVICE_STEP), eq(ImmutableList.copyOf(SERVICE_FIELDS)))).thenReturn(operationPaths);
    StepMeta input1Meta = mock(StepMeta.class);
    when(transMeta.findStep("Input 1")).thenReturn(input1Meta);
    when(serviceProvider.supportsStep(input1Meta)).thenReturn(true);
    List<PushDownOptimizationMeta> optimizationMetaList = service.apply(dataService);
    assertThat(optimizationMetaList, contains(allOf(hasProperty("stepName", equalTo("Input 1")), hasProperty("type", is(parameterGeneration)))));
    verify(parameterGeneration).setParameterName(anyString());
    verify(parameterGeneration).createFieldMapping("field2", "field2_origin");
}

Example 5 with StepFieldOperations

use of org.pentaho.metaverse.api.StepFieldOperations in project pdi-dataservice-server-plugin by pentaho.

the class AutoParameterGenerationService method apply.

@Override
public List<PushDownOptimizationMeta> apply(DataServiceMeta dataServiceMeta) {
    TransMeta transMeta = dataServiceMeta.getServiceTrans();
    LogChannelInterface logChannel = transMeta.getLogChannel() != null ? transMeta.getLogChannel() : LogChannel.GENERAL;
    try {
        String stepName = dataServiceMeta.getStepname();
        List<String> fields = Arrays.asList(transMeta.getStepFields(stepName).getFieldNames());
        Map<String, Set<List<StepFieldOperations>>> operationPaths = lineageClient.getOperationPaths(transMeta, stepName, fields);
        SourceLineageMap sourceLineageMap = SourceLineageMap.create(operationPaths).filterKeys(stepSupported(transMeta)).filterKeys(not(in(parametrizedSteps(dataServiceMeta)))).filter(fieldUnchanged());
        return generateOptimizationList(sourceLineageMap);
    } catch (Throwable e) {
        logChannel.logError("Unable to run Auto-Optimization", e);
        return Collections.emptyList();
    }
}