Examples with EndPoint io.cdap.cdap.api.lineage.field.EndPoint used on opensource projects

Example 16 with EndPoint

use of io.cdap.cdap.api.lineage.field.EndPoint in project cdap by caskdata.

the class FieldLineageInfo method computeAndValidateFieldLineageInfo.

private void computeAndValidateFieldLineageInfo(Collection<? extends Operation> operations) {
    Set<String> allOrigins = new HashSet<>();
    this.operationsMap = new HashMap<>();
    this.writeOperations = new HashSet<>();
    this.readOperations = new HashSet<>();
    this.operationOutgoingConnections = new HashMap<>();
    for (Operation operation : operations) {
        if (operationsMap.containsKey(operation.getName())) {
            throw new IllegalArgumentException(String.format("All operations provided for creating field " + "level lineage info must have unique names. " + "Operation name '%s' is repeated.", operation.getName()));
        }
        operationsMap.put(operation.getName(), operation);
        switch(operation.getType()) {
            case READ:
                ReadOperation read = (ReadOperation) operation;
                EndPoint source = read.getSource();
                if (source == null) {
                    throw new IllegalArgumentException(String.format("Source endpoint cannot be null for the read " + "operation '%s'.", read.getName()));
                }
                readOperations.add(read);
                break;
            case TRANSFORM:
                TransformOperation transform = (TransformOperation) operation;
                Set<String> origins = transform.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
                // for each origin corresponding to the input fields there is a connection from that origin to this operation
                for (String origin : origins) {
                    Set<Operation> connections = operationOutgoingConnections.computeIfAbsent(origin, k -> new HashSet<>());
                    connections.add(transform);
                }
                allOrigins.addAll(origins);
                if (transform.getOutputs().isEmpty()) {
                    dropTransforms.add(transform);
                }
                break;
            case WRITE:
                WriteOperation write = (WriteOperation) operation;
                EndPoint destination = write.getDestination();
                if (destination == null) {
                    throw new IllegalArgumentException(String.format("Destination endpoint cannot be null for the write " + "operation '%s'.", write.getName()));
                }
                origins = write.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
                // for each origin corresponding to the input fields there is a connection from that origin to this operation
                for (String origin : origins) {
                    Set<Operation> connections = operationOutgoingConnections.computeIfAbsent(origin, k -> new HashSet<>());
                    connections.add(write);
                }
                allOrigins.addAll(origins);
                writeOperations.add(write);
                break;
            default:
        }
    }
    Set<String> operationsWithNoOutgoingConnections = Sets.difference(operationsMap.keySet(), operationOutgoingConnections.keySet());
    // put empty set for operations with no outgoing connection rather than checking for null later
    for (String operation : operationsWithNoOutgoingConnections) {
        operationOutgoingConnections.put(operation, new HashSet<>());
    }
    if (readOperations.isEmpty()) {
        throw new IllegalArgumentException("Field level lineage requires at least one operation of type 'READ'.");
    }
    if (writeOperations.isEmpty()) {
        throw new IllegalArgumentException("Field level lineage requires at least one operation of type 'WRITE'.");
    }
    Sets.SetView<String> invalidOrigins = Sets.difference(allOrigins, operationsMap.keySet());
    if (!invalidOrigins.isEmpty()) {
        throw new IllegalArgumentException(String.format("No operation is associated with the origins '%s'.", invalidOrigins));
    }
}

Example 17 with EndPoint

use of io.cdap.cdap.api.lineage.field.EndPoint in project cdap by caskdata.

the class FieldLineageInfo method computeIncomingSummaryHelper.

/**
 * Helper method to compute the incoming summary
 *
 * @param currentOperation the operation being processed. Since we are processing incoming this operation is on the
 * left side if graph is imagined in horizontal orientation or this operation is the input to the to
 * previousOperation
 * @param previousOperation the previous operation which is processed and reside on right to the current operation if
 * the graph is imagined to be in horizontal orientation.
 * @param operationEndPointMap a map that contains the operation name to the final endpoint field it will generate,
 * this is used to track the path we already computed to ensure we do not do the same computation again
 */
private Set<EndPointField> computeIncomingSummaryHelper(Operation currentOperation, Operation previousOperation, Map<String, Set<EndPointField>> operationEndPointMap) {
    if (currentOperation.getType() == OperationType.READ) {
        // if current operation is of type READ, previous operation must be of type TRANSFORM or WRITE
        // get only the input fields from the previous operations for which the origin is current READ operation
        Set<InputField> inputFields = new HashSet<>();
        if (OperationType.WRITE == previousOperation.getType()) {
            WriteOperation previousWrite = (WriteOperation) previousOperation;
            inputFields = new HashSet<>(previousWrite.getInputs());
        } else if (OperationType.TRANSFORM == previousOperation.getType()) {
            TransformOperation previousTransform = (TransformOperation) previousOperation;
            inputFields = new HashSet<>(previousTransform.getInputs());
        }
        Set<EndPointField> sourceEndPointFields = new HashSet<>();
        // for all the input fields of the previous operation if the origin was current operation (remember we are
        // traversing backward)
        ReadOperation read = (ReadOperation) currentOperation;
        EndPoint source = read.getSource();
        for (InputField inputField : inputFields) {
            if (inputField.getOrigin().equals(currentOperation.getName())) {
                sourceEndPointFields.add(new EndPointField(source, inputField.getName()));
            }
        }
        // reached the end of graph unwind the recursive calls
        return sourceEndPointFields;
    }
    Set<EndPointField> relatedSources = new HashSet<>();
    // for transform we traverse backward in graph further through the inputs of the transform
    if (currentOperation.getType() == OperationType.TRANSFORM) {
        TransformOperation transform = (TransformOperation) currentOperation;
        // optimization to avoid repeating work if there are input fields with the same origin
        Set<String> transformOrigins = transform.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
        for (String transformOrigin : transformOrigins) {
            if (operationEndPointMap.containsKey(transformOrigin)) {
                relatedSources.addAll(operationEndPointMap.get(transformOrigin));
            } else {
                relatedSources.addAll(computeIncomingSummaryHelper(operationsMap.get(transformOrigin), currentOperation, operationEndPointMap));
            }
        }
        operationEndPointMap.put(currentOperation.getName(), relatedSources);
    }
    return relatedSources;
}

Example 18 with EndPoint

use of io.cdap.cdap.api.lineage.field.EndPoint in project cdap by caskdata.

the class FieldLineageInfoTest method testMultiSourceSingleDestinationWithoutMerge.

@Test
public void testMultiSourceSingleDestinationWithoutMerge() {
    // pRead: personFile -> (offset, body)
    // parse: body -> (id, name, address)
    // cRead: codeFile -> id
    // codeGen: (parse.id, cRead.id) -> id
    // sWrite: (codeGen.id, parse.name, parse.address) -> secureStore
    // iWrite: (parse.id, parse.name, parse.address) -> insecureStore
    EndPoint pEndPoint = EndPoint.of("ns", "personFile");
    EndPoint cEndPoint = EndPoint.of("ns", "codeFile");
    EndPoint sEndPoint = EndPoint.of("ns", "secureStore");
    EndPoint iEndPoint = EndPoint.of("ns", "insecureStore");
    ReadOperation pRead = new ReadOperation("pRead", "Reading from person file", pEndPoint, "offset", "body");
    ReadOperation cRead = new ReadOperation("cRead", "Reading from code file", cEndPoint, "id");
    TransformOperation parse = new TransformOperation("parse", "parsing body", Collections.singletonList(InputField.of("pRead", "body")), "id", "name", "address");
    TransformOperation codeGen = new TransformOperation("codeGen", "Generate secure code", Arrays.asList(InputField.of("parse", "id"), InputField.of("cRead", "id")), "id");
    WriteOperation sWrite = new WriteOperation("sWrite", "writing secure store", sEndPoint, Arrays.asList(InputField.of("codeGen", "id"), InputField.of("parse", "name"), InputField.of("parse", "address")));
    WriteOperation iWrite = new WriteOperation("iWrite", "writing insecure store", iEndPoint, Arrays.asList(InputField.of("parse", "id"), InputField.of("parse", "name"), InputField.of("parse", "address")));
    List<Operation> operations = new ArrayList<>();
    operations.add(pRead);
    operations.add(cRead);
    operations.add(parse);
    operations.add(codeGen);
    operations.add(sWrite);
    operations.add(iWrite);
    FieldLineageInfo fllInfo = new FieldLineageInfo(operations);
    Map<EndPoint, Set<String>> destinationFields = fllInfo.getDestinationFields();
    Assert.assertEquals(new HashSet<>(Arrays.asList("id", "name", "address")), destinationFields.get(sEndPoint));
    Assert.assertEquals(new HashSet<>(Arrays.asList("id", "name", "address")), destinationFields.get(iEndPoint));
    Assert.assertNull(destinationFields.get(pEndPoint));
    Map<EndPointField, Set<EndPointField>> incomingSummary = fllInfo.getIncomingSummary();
    Assert.assertEquals(6, incomingSummary.size());
    EndPointField expected = new EndPointField(pEndPoint, "body");
    Assert.assertEquals(1, incomingSummary.get(new EndPointField(iEndPoint, "id")).size());
    Assert.assertEquals(expected, incomingSummary.get(new EndPointField(iEndPoint, "id")).iterator().next());
    Assert.assertEquals(1, incomingSummary.get(new EndPointField(iEndPoint, "name")).size());
    Assert.assertEquals(expected, incomingSummary.get(new EndPointField(iEndPoint, "name")).iterator().next());
    Assert.assertEquals(1, incomingSummary.get(new EndPointField(iEndPoint, "address")).size());
    Assert.assertEquals(expected, incomingSummary.get(new EndPointField(iEndPoint, "address")).iterator().next());
    // name and address from secure endpoint also depends on the body field of pEndPoint
    Assert.assertEquals(1, incomingSummary.get(new EndPointField(sEndPoint, "name")).size());
    Assert.assertEquals(expected, incomingSummary.get(new EndPointField(sEndPoint, "name")).iterator().next());
    Assert.assertEquals(1, incomingSummary.get(new EndPointField(sEndPoint, "address")).size());
    Assert.assertEquals(expected, incomingSummary.get(new EndPointField(sEndPoint, "address")).iterator().next());
    // id of secure endpoint depends on both body field of pEndPoint and id field of cEndPoint
    Set<EndPointField> expectedSet = new HashSet<>();
    expectedSet.add(new EndPointField(pEndPoint, "body"));
    expectedSet.add(new EndPointField(cEndPoint, "id"));
    Assert.assertEquals(expectedSet, incomingSummary.get(new EndPointField(sEndPoint, "id")));
    Map<EndPointField, Set<EndPointField>> outgoingSummary = fllInfo.getOutgoingSummary();
    // outgoing summary will not contain offset but only body from pEndPoint and id from cEndPoint
    Assert.assertEquals(2, outgoingSummary.size());
    expectedSet = new HashSet<>();
    expectedSet.add(new EndPointField(iEndPoint, "id"));
    expectedSet.add(new EndPointField(iEndPoint, "name"));
    expectedSet.add(new EndPointField(iEndPoint, "address"));
    expectedSet.add(new EndPointField(sEndPoint, "id"));
    expectedSet.add(new EndPointField(sEndPoint, "name"));
    expectedSet.add(new EndPointField(sEndPoint, "address"));
    // body affects all fields from both secure and insecure endpoints
    Assert.assertEquals(expectedSet, outgoingSummary.get(new EndPointField(pEndPoint, "body")));
    expectedSet.clear();
    expectedSet.add(new EndPointField(sEndPoint, "id"));
    // id field of cEndPoint only affects id field of secure endpoint
    Assert.assertEquals(expectedSet, outgoingSummary.get(new EndPointField(cEndPoint, "id")));
    // Test incoming operations from all destination fields
    Set<Operation> inComingOperations = fllInfo.getIncomingOperationsForField(new EndPointField(iEndPoint, "id"));
    Set<Operation> expectedOperations = new HashSet<>();
    expectedOperations.add(iWrite);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, inComingOperations);
    inComingOperations = fllInfo.getIncomingOperationsForField(new EndPointField(iEndPoint, "name"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(iWrite);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(new FieldLineageInfo(expectedOperations), new FieldLineageInfo(inComingOperations));
    inComingOperations = fllInfo.getIncomingOperationsForField(new EndPointField(iEndPoint, "address"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(iWrite);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, inComingOperations);
    inComingOperations = fllInfo.getIncomingOperationsForField(new EndPointField(sEndPoint, "id"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(sWrite);
    expectedOperations.add(codeGen);
    expectedOperations.add(cRead);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, inComingOperations);
    inComingOperations = fllInfo.getIncomingOperationsForField(new EndPointField(sEndPoint, "name"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(sWrite);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, inComingOperations);
    inComingOperations = fllInfo.getIncomingOperationsForField(new EndPointField(sEndPoint, "address"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(sWrite);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, inComingOperations);
    // test outgoing operations for all source fields
    Set<Operation> outgoingOperations = fllInfo.getOutgoingOperationsForField(new EndPointField(pEndPoint, "offset"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, outgoingOperations);
    outgoingOperations = fllInfo.getOutgoingOperationsForField(new EndPointField(pEndPoint, "body"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(sWrite);
    expectedOperations.add(iWrite);
    expectedOperations.add(codeGen);
    expectedOperations.add(parse);
    expectedOperations.add(pRead);
    Assert.assertEquals(expectedOperations, outgoingOperations);
    outgoingOperations = fllInfo.getOutgoingOperationsForField(new EndPointField(cEndPoint, "id"));
    expectedOperations = new HashSet<>();
    expectedOperations.add(sWrite);
    expectedOperations.add(codeGen);
    expectedOperations.add(cRead);
    Assert.assertEquals(expectedOperations, outgoingOperations);
}

Example 19 with EndPoint

use of io.cdap.cdap.api.lineage.field.EndPoint in project cdap by caskdata.

the class FieldLineageInfoTest method testCycleWithNonExistentOperationNames.

@Test(expected = IllegalArgumentException.class)
public void testCycleWithNonExistentOperationNames() {
    EndPoint readEndPoint = EndPoint.of("ns", "file1");
    EndPoint writeEndPoint = EndPoint.of("ns", "file2");
    ReadOperation read = new ReadOperation("read", "read", readEndPoint, "offset", "body");
    TransformOperation parse = new TransformOperation("parse", "parse", Arrays.asList(InputField.of("read", "body"), InputField.of("normalize", "name"), InputField.of("nop1", "field1")), "name", "address");
    TransformOperation normalize = new TransformOperation("normalize", "normalize", Arrays.asList(InputField.of("parse", "name"), InputField.of("nop2", "field2")), "name");
    WriteOperation write = new WriteOperation("write", "writing to another file", writeEndPoint, Arrays.asList(InputField.of("normalize", "name"), InputField.of("parse", "address"), InputField.of("nop3", "field3")));
    List<Operation> operations = new ArrayList<>();
    operations.add(parse);
    operations.add(read);
    operations.add(normalize);
    operations.add(write);
    FieldLineageInfo.getTopologicallySortedOperations(new HashSet<>(operations));
}

Example 20 with EndPoint

use of io.cdap.cdap.api.lineage.field.EndPoint in project cdap by caskdata.

the class FieldLineageInfoTest method testMultiSourceDroppedFields.

@Test
public void testMultiSourceDroppedFields() {
    ReadOperation read = new ReadOperation("read", "some read", EndPoint.of("endpoint1"), "first_name", "last_name", "social");
    TransformOperation combineNames = new TransformOperation("combineNames", "combine names", Arrays.asList(InputField.of("read", "first_name"), InputField.of("read", "last_name")), "full_name");
    TransformOperation dropSocial = new TransformOperation("dropSocial", "drop social", Collections.singletonList(InputField.of("read", "social")));
    WriteOperation write = new WriteOperation("write", "write data", EndPoint.of("endpoint2"), Collections.singletonList(InputField.of("combineNames", "full_name")));
    Set<Operation> operations = Sets.newHashSet(read, write, combineNames, dropSocial);
    FieldLineageInfo info1 = new FieldLineageInfo(operations);
    EndPoint ep1 = EndPoint.of("endpoint1");
    EndPoint ep2 = EndPoint.of("endpoint2");
    Map<EndPointField, Set<EndPointField>> expectedOutgoingSummary = new HashMap<>();
    expectedOutgoingSummary.put(new EndPointField(ep1, "first_name"), Collections.singleton(new EndPointField(ep2, "full_name")));
    expectedOutgoingSummary.put(new EndPointField(ep1, "last_name"), Collections.singleton(new EndPointField(ep2, "full_name")));
    expectedOutgoingSummary.put(new EndPointField(ep1, "social"), Collections.singleton(FieldLineageInfo.NULL_EPF));
    Assert.assertEquals(expectedOutgoingSummary, info1.getOutgoingSummary());
    Map<EndPointField, Set<EndPointField>> expectedIncomingSummary = new HashMap<>();
    expectedIncomingSummary.put(new EndPointField(ep2, "full_name"), Sets.newHashSet(new EndPointField(ep1, "first_name"), new EndPointField(ep1, "last_name")));
    Assert.assertEquals(expectedIncomingSummary, info1.getIncomingSummary());
}