Examples with StepField org.pentaho.metaverse.api.StepField used on opensource projects

Example 26 with StepField

use of org.pentaho.metaverse.api.StepField in project pentaho-metaverse by pentaho.

the class StepAnalyzerTest method testMapChange_originalFieldIsTransient.

@Test
public void testMapChange_originalFieldIsTransient() throws Exception {
    doReturn(outputs).when(analyzer).getOutputs();
    doReturn(inputs).when(analyzer).getInputs();
    IMetaverseNode transientNode = mock(IMetaverseNode.class);
    String shouldBeNull = null;
    doReturn(transientNode).when(analyzer).createOutputFieldNode(any(IAnalysisContext.class), any(ValueMetaInterface.class), eq(shouldBeNull), eq(DictionaryConst.NODE_TYPE_TRANS_FIELD));
    // zip is not in the inputs or outputs, it must have been a temporary fields used internally by step.
    StepField original = new StepField(null, "zip");
    StepField changed = new StepField("nextStep", "address");
    ComponentDerivationRecord cdr = new ComponentDerivationRecord(original, changed);
    analyzer.mapChange(cdr);
    verify(builder).addLink(rootNode, DictionaryConst.LINK_TRANSIENT, transientNode);
    verify(builder).addLink(rootNode, DictionaryConst.LINK_USES, transientNode);
    verify(analyzer).linkChangeNodes(eq(transientNode), any(IMetaverseNode.class));
}

Example 27 with StepField

use of org.pentaho.metaverse.api.StepField in project pentaho-metaverse by pentaho.

the class SelectValuesStepAnalyzer method getUsedFields.

@Override
protected Set<StepField> getUsedFields(SelectValuesMeta meta) {
    Set<StepField> usedFields = new HashSet<>();
    String[] fieldNames = meta.getSelectName();
    for (String fieldName : fieldNames) {
        usedFields.addAll(createStepFields(fieldName, getInputs()));
    }
    SelectMetadataChange[] selectMetadataChanges = meta.getMeta();
    for (SelectMetadataChange selectMetadataChange : selectMetadataChanges) {
        usedFields.addAll(createStepFields(selectMetadataChange.getName(), getInputs()));
    }
    return usedFields;
}

Example 28 with StepField

use of org.pentaho.metaverse.api.StepField in project pentaho-metaverse by pentaho.

the class TableOutputStepAnalyzer method getChangeRecords.

@Override
public Set<ComponentDerivationRecord> getChangeRecords(TableOutputMeta meta) throws MetaverseAnalyzerException {
    Set<ComponentDerivationRecord> changes = new HashSet<>();
    String[] tableFields = meta.getFieldDatabase();
    String[] streamFields = meta.getFieldStream();
    if (getInputs() != null) {
        Set<String> stepNames = getInputs().getStepNames();
        for (int i = 0; i < tableFields.length; i++) {
            String tableField = tableFields[i];
            String streamField = streamFields[i];
            for (String stepName : stepNames) {
                if (!RESOURCE.equals(stepName)) {
                    StepField inputField = new StepField(stepName, streamField);
                    StepField outField = new StepField(RESOURCE, tableField);
                    ComponentDerivationRecord change = new ComponentDerivationRecord(inputField, outField);
                    changes.add(change);
                }
            }
        }
    }
    return changes;
}

Example 29 with StepField

use of org.pentaho.metaverse.api.StepField in project pentaho-metaverse by pentaho.

the class LineageClient method getOriginSteps.

/**
 * Finds the step(s) in the given transformation that created the given field, with respect to the given target step.
 * This means if a field has been renamed or derived from another field from another step, then the lineage graph
 * is traversed back from the target step to determine which steps contributed to the field in the target step.
 * This differs from getCreatorSteps() as the lineage graph traversal will not stop with a "creates" relationship;
 * rather, this method will traverse other relationships ("uses", "derives", e.g.) to find the actual origin fields
 * that comprise the final field in the target step.
 *
 * @param transMeta      a reference to a transformation's metadata
 * @param targetStepName the target step name associated with the given field names
 * @param fieldNames     a collection of field names associated with the target step, for which to find the step(s)
 *                       and field(s) that contributed to those fields
 * @return a map from target field name to step-field objects, where each step has created a field with
 * the returned name, and that field has contributed in some way to the specified target field.
 * @throws MetaverseException if an error occurred while finding the origin steps
 */
@Override
public Map<String, Set<StepField>> getOriginSteps(TransMeta transMeta, String targetStepName, Collection<String> fieldNames) throws MetaverseException {
    Map<String, Set<StepField>> originStepsMap = new HashMap<>();
    try {
        Future<Graph> lineageGraphTask = LineageGraphMap.getInstance().get(transMeta);
        if (lineageGraphTask != null) {
            Graph lineageGraph = lineageGraphTask.get();
            List<Vertex> targetFields = getTargetFields(lineageGraph, targetStepName, fieldNames);
            GremlinPipeline pipe = getOriginStepsPipe(targetFields);
            List<List<Vertex>> pathList = pipe.toList();
            if (pathList != null) {
                for (List<Vertex> path : pathList) {
                    // Transform each path of vertices into a "path" of StepFieldOperations objects (basically save off
                    // properties of each vertex into a new list)
                    String targetField = path.get(0).getProperty(DictionaryConst.PROPERTY_NAME);
                    Set<StepField> pathSet = originStepsMap.get(targetField);
                    if (pathSet == null) {
                        pathSet = new HashSet<>();
                        originStepsMap.put(targetField, pathSet);
                    }
                    Vertex v = path.get(path.size() - 1);
                    Map<String, String> stepField = STEPFIELDOPS_PIPE_FUNC.compute(v);
                    String stepName = stepField.get("stepName");
                    String fieldName = stepField.get("fieldName");
                    pathSet.add(new StepField(stepName, fieldName));
                }
            }
        }
    } catch (Exception e) {
        throw new MetaverseException(e);
    }
    return originStepsMap;
}

Example 30 with StepField

use of org.pentaho.metaverse.api.StepField in project pentaho-metaverse by pentaho.

the class TextFileInputStepAnalyzerTest method testGetUsedFields_fileNameFromField.

@Test
public void testGetUsedFields_fileNameFromField() throws Exception {
    when(meta.isAcceptingFilenames()).thenReturn(true);
    when(meta.getAcceptingField()).thenReturn("filename");
    when(meta.getAcceptingStepName()).thenReturn("previousStep");
    Set<StepField> usedFields = analyzer.getUsedFields(meta);
    assertNotNull(usedFields);
    assertEquals(1, usedFields.size());
    StepField used = usedFields.iterator().next();
    assertEquals("previousStep", used.getStepName());
    assertEquals("filename", used.getFieldName());
}