Examples with FieldTransformOperation io.cdap.cdap.etl.api.lineage.field.FieldTransformOperation used on opensource projects

Example 51 with FieldTransformOperation

use of io.cdap.cdap.etl.api.lineage.field.FieldTransformOperation in project hydrator-plugins by cdapio.

the class JoinFieldLineageTest method compareOperations.

/**
 * Can't compare directly because input and output fields are Lists, but the order is not actually guaranteed.
 * So need to compare manually using sets instead of lists.
 */
private void compareOperations(List<FieldTransformOperation> expected, List<FieldOperation> actual) {
    Assert.assertEquals(expected.size(), actual.size());
    Iterator<FieldTransformOperation> expectedIter = expected.iterator();
    Iterator<FieldOperation> actualIter = actual.iterator();
    while (expectedIter.hasNext()) {
        FieldTransformOperation expectedOp = expectedIter.next();
        FieldOperation actualOp = actualIter.next();
        Assert.assertEquals(OperationType.TRANSFORM, actualOp.getType());
        FieldTransformOperation actualTransformOp = (FieldTransformOperation) actualOp;
        Assert.assertEquals(expectedOp.getName(), actualTransformOp.getName());
        Assert.assertEquals(new HashSet<>(expectedOp.getInputFields()), new HashSet<>(actualTransformOp.getInputFields()));
        Assert.assertEquals(new HashSet<>(expectedOp.getOutputFields()), new HashSet<>(actualTransformOp.getOutputFields()));
    }
}

Example 52 with FieldTransformOperation

use of io.cdap.cdap.etl.api.lineage.field.FieldTransformOperation in project hydrator-plugins by cdapio.

the class ValueMapper method prepareRun.

@Override
public void prepareRun(StageSubmitterContext context) throws Exception {
    super.prepareRun(context);
    if (context.getInputSchema() == null || context.getInputSchema().getFields() == null) {
        return;
    }
    parseConfiguration(this.config, context.getFailureCollector());
    // After extracting the mappings, store a list of operations containing identity transforms for every output
    // field also present in the mappings list.
    List<String> mappedFields = TransformLineageRecorderUtils.getFields(context.getInputSchema()).stream().filter(mappingValues::containsKey).collect(Collectors.toList());
    List<String> identityFields = TransformLineageRecorderUtils.getFields(context.getInputSchema());
    identityFields.removeAll(mappedFields);
    List<FieldOperation> output = new ArrayList<>();
    output.addAll(mappedFields.stream().map(sourceFieldName -> new FieldTransformOperation("mapValueOf" + sourceFieldName, "Mapped values of fields based on the lookup table.", Collections.singletonList(sourceFieldName), mappingValues.get(sourceFieldName).getDefaultValue())).collect(Collectors.toList()));
    output.addAll(TransformLineageRecorderUtils.generateOneToOnes(identityFields, "identity", TransformLineageRecorderUtils.IDENTITY_TRANSFORM_DESCRIPTION));
    context.record(output);
}

Example 53 with FieldTransformOperation

use of io.cdap.cdap.etl.api.lineage.field.FieldTransformOperation in project hydrator-plugins by cdapio.

the class JavaScriptTransform method prepareRun.

@Override
public void prepareRun(StageSubmitterContext context) throws Exception {
    super.prepareRun(context);
    List<String> inputFields = new ArrayList<>();
    List<String> outputFields = new ArrayList<>();
    Schema inputSchema = context.getInputSchema();
    if (SchemaValidator.canRecordLineage(inputSchema, "input")) {
        // noinspection ConstantConditions
        inputFields = inputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList());
    }
    Schema outputSchema = context.getOutputSchema();
    if (SchemaValidator.canRecordLineage(outputSchema, "output")) {
        // noinspection ConstantConditions
        outputFields = outputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList());
    }
    FieldOperation dataPrepOperation = new FieldTransformOperation("JavaScript", config.script, inputFields, outputFields);
    context.record(Collections.singletonList(dataPrepOperation));
}