Examples with ComponentDerivationRecord org.pentaho.metaverse.api.analyzer.kettle.ComponentDerivationRecord used on opensource projects

Example 1 with ComponentDerivationRecord

use of org.pentaho.metaverse.api.analyzer.kettle.ComponentDerivationRecord in project pentaho-kettle by pentaho.

the class GetXMLDataStepAnalyzerTest method testGetChangeRecords.

@Test
public void testGetChangeRecords() throws Exception {
    when(meta.isInFields()).thenReturn(true);
    when(meta.getIsAFile()).thenReturn(false);
    when(meta.isReadUrl()).thenReturn(false);
    when(meta.getXMLField()).thenReturn("xml");
    analyzer.setBaseStepMeta(meta);
    GetXMLDataField[] fields = new GetXMLDataField[2];
    GetXMLDataField field1 = new GetXMLDataField("name");
    GetXMLDataField field2 = new GetXMLDataField("age");
    field1.setXPath("field1/xpath");
    field2.setElementType(1);
    field1.setResultType(1);
    field2.setRepeated(true);
    fields[0] = field1;
    fields[1] = field2;
    when(meta.getInputFields()).thenReturn(fields);
    StepNodes inputs = new StepNodes();
    inputs.addNode("previousStep", "xml", node);
    doReturn(inputs).when(analyzer).getInputs();
    Set<ComponentDerivationRecord> changeRecords = analyzer.getChangeRecords(meta);
    assertNotNull(changeRecords);
    assertEquals(2, changeRecords.size());
}

Example 2 with ComponentDerivationRecord

use of org.pentaho.metaverse.api.analyzer.kettle.ComponentDerivationRecord in project pentaho-metaverse by pentaho.

the class MergeJoinStepAnalyzer method getChangeRecords.

/**
 * Identify the name collision renames and add change records for them.
 *
 * example: join fields in both input steps named COUNTRY. the second (right) field gets renamed with a suffix
 * on the way out of the step. You end up with COUNTRY (from the left) & COUNTRY_1 (from the right)
 *
 * @param meta
 * @return
 * @throws MetaverseAnalyzerException
 */
@Override
public Set<ComponentDerivationRecord> getChangeRecords(MergeJoinMeta meta) throws MetaverseAnalyzerException {
    Set<ComponentDerivationRecord> changeRecords = new HashSet<>();
    String[] prevStepNames = parentTransMeta.getPrevStepNames(getStepName());
    if (getOutputs() != null) {
        Set<StepField> outputFields = getOutputs().getFieldNames();
        for (StepField outputField : outputFields) {
            if (outputField.getFieldName().matches(".*_\\d*$")) {
                String unsuffixName = outputField.getFieldName().replaceAll("_\\d*$", "");
                StepField rightSideInputField = new StepField(prevStepNames[1], unsuffixName);
                if (!isPassthrough(rightSideInputField)) {
                    ComponentDerivationRecord renameFieldRecord = new ComponentDerivationRecord(rightSideInputField, outputField, ChangeType.METADATA);
                    renameFieldRecord.addOperation(Operation.getRenameOperation());
                    changeRecords.add(renameFieldRecord);
                }
            }
        }
    }
    return changeRecords;
}

Example 3 with ComponentDerivationRecord

use of org.pentaho.metaverse.api.analyzer.kettle.ComponentDerivationRecord in project pentaho-metaverse by pentaho.

the class SelectValuesStepAnalyzer method getChangeRecords.

@Override
public Set<ComponentDerivationRecord> getChangeRecords(SelectValuesMeta selectValuesMeta) throws MetaverseAnalyzerException {
    validateState(null, selectValuesMeta);
    Set<ComponentDerivationRecord> changeRecords = new HashSet<ComponentDerivationRecord>();
    String inputFieldName;
    String outputFieldName;
    ComponentDerivationRecord changeRecord;
    // Process the fields/tabs in the same order as the real step does
    if (!Const.isEmpty(selectValuesMeta.getSelectName())) {
        String[] fieldNames = selectValuesMeta.getSelectName();
        String[] fieldRenames = selectValuesMeta.getSelectRename();
        int[] fieldLength = selectValuesMeta.getSelectLength();
        int[] fieldPrecision = selectValuesMeta.getSelectPrecision();
        for (int i = 0; i < fieldNames.length; i++) {
            inputFieldName = fieldNames[i];
            outputFieldName = fieldRenames[i];
            changeRecord = new ComponentDerivationRecord(inputFieldName, outputFieldName == null ? inputFieldName : outputFieldName, ChangeType.METADATA);
            Set<String> metadataChangedFields = new HashSet<String>();
            if (inputFieldName != null && outputFieldName != null && !inputFieldName.equals(outputFieldName)) {
                metadataChangedFields.add("name");
            }
            // Check for changes in field length
            if (fieldLength != null && fieldLength[i] != NOT_CHANGED) {
                metadataChangedFields.add("length");
            }
            // Check for changes in field precision
            if (fieldPrecision != null && fieldPrecision[i] != NOT_CHANGED) {
                metadataChangedFields.add("precision");
            }
            if (!metadataChangedFields.isEmpty()) {
                // Add all the changed metadata fields as a single operation
                changeRecord.addOperation(new Operation(DictionaryConst.PROPERTY_MODIFIED, StringUtils.join(metadataChangedFields, ",")));
            }
            changeRecords.add(changeRecord);
        }
    }
    if (!Const.isEmpty(selectValuesMeta.getMeta())) {
        String[] prevStepNames = parentTransMeta.getPrevStepNames(getStepName());
        SelectMetadataChange[] metadataChanges = selectValuesMeta.getMeta();
        if (metadataChanges != null) {
            for (SelectMetadataChange metadataChange : metadataChanges) {
                inputFieldName = metadataChange.getName();
                outputFieldName = metadataChange.getRename();
                changeRecord = new ComponentDerivationRecord(inputFieldName, outputFieldName == null ? inputFieldName : outputFieldName, ChangeType.METADATA);
                Set<String> metadataChangedFields = new HashSet<String>();
                // NOTE: We use equalsIgnoreCase instead of equals because that's how Select Values currently works
                if (inputFieldName != null && outputFieldName != null && !inputFieldName.equalsIgnoreCase(outputFieldName)) {
                    metadataChangedFields.add("name");
                }
                // Get the ValueMetaInterface for the input field, to determine if any of its metadata has changed
                if (prevFields == null) {
                    prevFields = getInputFields(selectValuesMeta);
                    if (prevFields == null) {
                        log.warn(Messages.getString("WARNING.CannotDetermineFieldType", inputFieldName));
                        continue;
                    }
                }
                RowMetaInterface rowMetaInterface = prevFields.get(prevStepNames[0]);
                ValueMetaInterface inputFieldValueMeta = null;
                if (rowMetaInterface == null) {
                    log.warn(Messages.getString("WARNING.CannotDetermineFieldType", inputFieldName));
                    continue;
                }
                inputFieldValueMeta = rowMetaInterface.searchValueMeta(inputFieldName);
                if (inputFieldValueMeta == null) {
                    log.warn(Messages.getString("WARNING.CannotDetermineFieldType", inputFieldName));
                    continue;
                }
                // Check for changes in field type
                if (inputFieldValueMeta.getType() != metadataChange.getType()) {
                    metadataChangedFields.add("type");
                }
                // Check for changes in field length
                if (metadataChange.getLength() != NOT_CHANGED) {
                    metadataChangedFields.add("length");
                }
                // Check for changes in field precision
                if (metadataChange.getPrecision() != NOT_CHANGED) {
                    metadataChangedFields.add("precision");
                }
                // Check for changes in storage type (binary to string, e.g.)
                if ((metadataChange.getStorageType() != -1) && (inputFieldValueMeta.getStorageType() != metadataChange.getStorageType())) {
                    metadataChangedFields.add("storagetype");
                }
                // Check for changes in conversion mask
                if ((metadataChange.getConversionMask() != null) && (inputFieldValueMeta.getConversionMask() == null || !inputFieldValueMeta.getConversionMask().equals(metadataChange.getConversionMask()))) {
                    metadataChangedFields.add("conversionmask");
                }
                // Check for changes in date format leniency
                if (inputFieldValueMeta.isDateFormatLenient() != metadataChange.isDateFormatLenient()) {
                    metadataChangedFields.add("dateformatlenient");
                }
                // Check for changes in date format locale
                if ((metadataChange.getDateFormatLocale() != null) && (inputFieldValueMeta.getDateFormatLocale() == null || !inputFieldValueMeta.getDateFormatLocale().toString().equals(metadataChange.getDateFormatLocale()))) {
                    metadataChangedFields.add("datelocale");
                }
                // Check for changes in date format locale
                if ((metadataChange.getDateFormatTimeZone() != null) && (inputFieldValueMeta.getDateFormatTimeZone() == null || !inputFieldValueMeta.getDateFormatTimeZone().toString().equals(metadataChange.getDateFormatTimeZone()))) {
                    metadataChangedFields.add("datetimezone");
                }
                // Check for changes in date format locale
                if (inputFieldValueMeta.isLenientStringToNumber() != metadataChange.isLenientStringToNumber()) {
                    metadataChangedFields.add("lenientnumberconversion");
                }
                // Check for changes in encoding
                if ((metadataChange.getDateFormatTimeZone() != null) && (inputFieldValueMeta.getStringEncoding() == null || !inputFieldValueMeta.getStringEncoding().equals(metadataChange.getDateFormatTimeZone()))) {
                    metadataChangedFields.add("encoding");
                }
                // Check for changes in encoding
                if ((metadataChange.getDecimalSymbol() != null) && (inputFieldValueMeta.getDecimalSymbol() == null || !inputFieldValueMeta.getDecimalSymbol().equals(metadataChange.getDecimalSymbol()))) {
                    metadataChangedFields.add("decimalsymbol");
                }
                // Check for changes in encoding
                if ((metadataChange.getGroupingSymbol() != null) && (inputFieldValueMeta.getGroupingSymbol() == null || !inputFieldValueMeta.getGroupingSymbol().equals(metadataChange.getGroupingSymbol()))) {
                    metadataChangedFields.add("groupsymbol");
                }
                // Check for changes in encoding
                if ((metadataChange.getCurrencySymbol() != null) && (inputFieldValueMeta.getCurrencySymbol() == null || !inputFieldValueMeta.getCurrencySymbol().equals(metadataChange.getCurrencySymbol()))) {
                    metadataChangedFields.add("currencysymbol");
                }
                if (!metadataChangedFields.isEmpty()) {
                    // Add all the changed metadata fields as a single operation
                    changeRecord.addOperation(new Operation(DictionaryConst.PROPERTY_MODIFIED, StringUtils.join(metadataChangedFields, ",")));
                }
                changeRecords.add(changeRecord);
            }
        }
    }
    return changeRecords;
}

Example 4 with ComponentDerivationRecord

use of org.pentaho.metaverse.api.analyzer.kettle.ComponentDerivationRecord in project pentaho-metaverse by pentaho.

the class SplitFieldsStepAnalyzer method getChangeRecords.

@Override
public Set<ComponentDerivationRecord> getChangeRecords(FieldSplitterMeta meta) throws MetaverseAnalyzerException {
    String originalField = meta.getSplitField();
    Set<ComponentDerivationRecord> changeRecords = new HashSet<ComponentDerivationRecord>();
    String[] fieldNames = meta.getFieldName();
    if (!Const.isEmpty(fieldNames)) {
        for (int i = 0; i < meta.getFieldName().length; i++) {
            ComponentDerivationRecord cdr = new ComponentDerivationRecord(originalField, fieldNames[i], ChangeType.DATA);
            cdr.addOperation(new Operation(Operation.MAPPING_CATEGORY, ChangeType.DATA, fieldNames[i], "Token " + i + " of split string"));
            changeRecords.add(cdr);
        }
    }
    return changeRecords;
}

Example 5 with ComponentDerivationRecord

use of org.pentaho.metaverse.api.analyzer.kettle.ComponentDerivationRecord in project pentaho-metaverse by pentaho.

the class StringsCutStepAnalyzer method getChangeRecords.

@Override
public Set<ComponentDerivationRecord> getChangeRecords(StringCutMeta meta) throws MetaverseAnalyzerException {
    Set<ComponentDerivationRecord> changeRecords = new HashSet<>();
    for (int i = 0; i < meta.getFieldInStream().length; i++) {
        String fieldInString = meta.getFieldInStream()[i];
        String fieldOutString = meta.getFieldOutStream()[i];
        if (fieldOutString == null || fieldOutString.length() < 1) {
            fieldOutString = fieldInString;
        }
        ComponentDerivationRecord changeRecord = new ComponentDerivationRecord(fieldInString, fieldOutString, ChangeType.DATA);
        String changeOperation = fieldInString + " -> [ substring [ " + meta.getCutFrom()[i] + ", " + meta.getCutTo()[i] + " ] ] -> " + fieldOutString;
        changeRecord.addOperation(new Operation(Operation.CALC_CATEGORY, ChangeType.DATA, DictionaryConst.PROPERTY_TRANSFORMS, changeOperation));
        changeRecords.add(changeRecord);
    }
    return changeRecords;
}