use of io.cdap.cdap.api.lineage.field.WriteOperation in project cdap by caskdata.
the class FieldLineageInfoTest method testInvalidOperations.
@Test
public void testInvalidOperations() {
ReadOperation read = new ReadOperation("read", "some read", EndPoint.of("endpoint1"), "offset", "body");
TransformOperation parse = new TransformOperation("parse", "parse body", Collections.singletonList(InputField.of("read", "body")), "name", "address");
WriteOperation write = new WriteOperation("write", "write data", EndPoint.of("ns", "endpoint2"), Arrays.asList(InputField.of("read", "offset"), InputField.of("parse", "name"), InputField.of("parse", "body")));
List<Operation> operations = new ArrayList<>();
operations.add(parse);
operations.add(write);
try {
// Create info without read operation
FieldLineageInfo info = new FieldLineageInfo(operations);
Assert.fail("Field lineage info creation should fail since no read operation is specified.");
} catch (IllegalArgumentException e) {
String msg = "Field level lineage requires at least one operation of type 'READ'.";
Assert.assertEquals(msg, e.getMessage());
}
operations.clear();
operations.add(read);
operations.add(parse);
try {
// Create info without write operation
FieldLineageInfo info = new FieldLineageInfo(operations);
Assert.fail("Field lineage info creation should fail since no write operation is specified.");
} catch (IllegalArgumentException e) {
String msg = "Field level lineage requires at least one operation of type 'WRITE'.";
Assert.assertEquals(msg, e.getMessage());
}
WriteOperation duplicateWrite = new WriteOperation("write", "write data", EndPoint.of("ns", "endpoint3"), Arrays.asList(InputField.of("read", "offset"), InputField.of("parse", "name"), InputField.of("parse", "body")));
operations.add(write);
operations.add(duplicateWrite);
try {
// Create info with non-unique operation names
FieldLineageInfo info = new FieldLineageInfo(operations);
Assert.fail("Field lineage info creation should fail since operation name 'write' is repeated.");
} catch (IllegalArgumentException e) {
String msg = "Operation name 'write' is repeated";
Assert.assertTrue(e.getMessage().contains(msg));
}
operations.clear();
TransformOperation invalidOrigin = new TransformOperation("anotherparse", "parse body", Arrays.asList(InputField.of("invalid", "body"), InputField.of("anotherinvalid", "body")), "name", "address");
operations.add(read);
operations.add(parse);
operations.add(write);
operations.add(invalidOrigin);
try {
// Create info without invalid origins
FieldLineageInfo info = new FieldLineageInfo(operations);
Assert.fail("Field lineage info creation should fail since operation with name 'invalid' " + "and 'anotherinvalid' do not exist.");
} catch (IllegalArgumentException e) {
String msg = "No operation is associated with the origins '[invalid, anotherinvalid]'.";
Assert.assertEquals(msg, e.getMessage());
}
}
use of io.cdap.cdap.api.lineage.field.WriteOperation in project cdap by caskdata.
the class FieldLineageInfo method populateSourcesAndDestinations.
private void populateSourcesAndDestinations() {
sources = new HashSet<>();
destinations = new HashSet<>();
for (Operation operation : operations) {
if (OperationType.READ == operation.getType()) {
ReadOperation read = (ReadOperation) operation;
sources.add(read.getSource());
} else if (OperationType.WRITE == operation.getType()) {
WriteOperation write = (WriteOperation) operation;
destinations.add(write.getDestination());
}
}
}
use of io.cdap.cdap.api.lineage.field.WriteOperation in project cdap by caskdata.
the class FieldLineageInfo method computeAndValidateFieldLineageInfo.
private void computeAndValidateFieldLineageInfo(Collection<? extends Operation> operations) {
Set<String> allOrigins = new HashSet<>();
this.operationsMap = new HashMap<>();
this.writeOperations = new HashSet<>();
this.readOperations = new HashSet<>();
this.operationOutgoingConnections = new HashMap<>();
for (Operation operation : operations) {
if (operationsMap.containsKey(operation.getName())) {
throw new IllegalArgumentException(String.format("All operations provided for creating field " + "level lineage info must have unique names. " + "Operation name '%s' is repeated.", operation.getName()));
}
operationsMap.put(operation.getName(), operation);
switch(operation.getType()) {
case READ:
ReadOperation read = (ReadOperation) operation;
EndPoint source = read.getSource();
if (source == null) {
throw new IllegalArgumentException(String.format("Source endpoint cannot be null for the read " + "operation '%s'.", read.getName()));
}
readOperations.add(read);
break;
case TRANSFORM:
TransformOperation transform = (TransformOperation) operation;
Set<String> origins = transform.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
// for each origin corresponding to the input fields there is a connection from that origin to this operation
for (String origin : origins) {
Set<Operation> connections = operationOutgoingConnections.computeIfAbsent(origin, k -> new HashSet<>());
connections.add(transform);
}
allOrigins.addAll(origins);
if (transform.getOutputs().isEmpty()) {
dropTransforms.add(transform);
}
break;
case WRITE:
WriteOperation write = (WriteOperation) operation;
EndPoint destination = write.getDestination();
if (destination == null) {
throw new IllegalArgumentException(String.format("Destination endpoint cannot be null for the write " + "operation '%s'.", write.getName()));
}
origins = write.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
// for each origin corresponding to the input fields there is a connection from that origin to this operation
for (String origin : origins) {
Set<Operation> connections = operationOutgoingConnections.computeIfAbsent(origin, k -> new HashSet<>());
connections.add(write);
}
allOrigins.addAll(origins);
writeOperations.add(write);
break;
default:
}
}
Set<String> operationsWithNoOutgoingConnections = Sets.difference(operationsMap.keySet(), operationOutgoingConnections.keySet());
// put empty set for operations with no outgoing connection rather than checking for null later
for (String operation : operationsWithNoOutgoingConnections) {
operationOutgoingConnections.put(operation, new HashSet<>());
}
if (readOperations.isEmpty()) {
throw new IllegalArgumentException("Field level lineage requires at least one operation of type 'READ'.");
}
if (writeOperations.isEmpty()) {
throw new IllegalArgumentException("Field level lineage requires at least one operation of type 'WRITE'.");
}
Sets.SetView<String> invalidOrigins = Sets.difference(allOrigins, operationsMap.keySet());
if (!invalidOrigins.isEmpty()) {
throw new IllegalArgumentException(String.format("No operation is associated with the origins '%s'.", invalidOrigins));
}
}
use of io.cdap.cdap.api.lineage.field.WriteOperation in project cdap by caskdata.
the class FieldLineageInfo method computeIncomingSummaryHelper.
/**
* Helper method to compute the incoming summary
*
* @param currentOperation the operation being processed. Since we are processing incoming this operation is on the
* left side if graph is imagined in horizontal orientation or this operation is the input to the to
* previousOperation
* @param previousOperation the previous operation which is processed and reside on right to the current operation if
* the graph is imagined to be in horizontal orientation.
* @param operationEndPointMap a map that contains the operation name to the final endpoint field it will generate,
* this is used to track the path we already computed to ensure we do not do the same computation again
*/
private Set<EndPointField> computeIncomingSummaryHelper(Operation currentOperation, Operation previousOperation, Map<String, Set<EndPointField>> operationEndPointMap) {
if (currentOperation.getType() == OperationType.READ) {
// if current operation is of type READ, previous operation must be of type TRANSFORM or WRITE
// get only the input fields from the previous operations for which the origin is current READ operation
Set<InputField> inputFields = new HashSet<>();
if (OperationType.WRITE == previousOperation.getType()) {
WriteOperation previousWrite = (WriteOperation) previousOperation;
inputFields = new HashSet<>(previousWrite.getInputs());
} else if (OperationType.TRANSFORM == previousOperation.getType()) {
TransformOperation previousTransform = (TransformOperation) previousOperation;
inputFields = new HashSet<>(previousTransform.getInputs());
}
Set<EndPointField> sourceEndPointFields = new HashSet<>();
// for all the input fields of the previous operation if the origin was current operation (remember we are
// traversing backward)
ReadOperation read = (ReadOperation) currentOperation;
EndPoint source = read.getSource();
for (InputField inputField : inputFields) {
if (inputField.getOrigin().equals(currentOperation.getName())) {
sourceEndPointFields.add(new EndPointField(source, inputField.getName()));
}
}
// reached the end of graph unwind the recursive calls
return sourceEndPointFields;
}
Set<EndPointField> relatedSources = new HashSet<>();
// for transform we traverse backward in graph further through the inputs of the transform
if (currentOperation.getType() == OperationType.TRANSFORM) {
TransformOperation transform = (TransformOperation) currentOperation;
// optimization to avoid repeating work if there are input fields with the same origin
Set<String> transformOrigins = transform.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
for (String transformOrigin : transformOrigins) {
if (operationEndPointMap.containsKey(transformOrigin)) {
relatedSources.addAll(operationEndPointMap.get(transformOrigin));
} else {
relatedSources.addAll(computeIncomingSummaryHelper(operationsMap.get(transformOrigin), currentOperation, operationEndPointMap));
}
}
operationEndPointMap.put(currentOperation.getName(), relatedSources);
}
return relatedSources;
}
use of io.cdap.cdap.api.lineage.field.WriteOperation in project cdap by caskdata.
the class FieldLineageInfo method computeIncomingSummary.
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() {
if (writeOperations == null) {
computeAndValidateFieldLineageInfo(this.operations);
}
Map<String, Set<EndPointField>> operationEndPointMap = new HashMap<>();
Map<EndPointField, Set<EndPointField>> summary = new HashMap<>();
for (WriteOperation write : writeOperations) {
List<InputField> inputs = write.getInputs();
for (InputField input : inputs) {
EndPointField dest = new EndPointField(write.getDestination(), input.getName());
Set<EndPointField> fields = summary.computeIfAbsent(dest, k -> new HashSet<>());
if (operationEndPointMap.containsKey(input.getOrigin())) {
fields.addAll(operationEndPointMap.get(input.getOrigin()));
} else {
// handle a special case for read -> write
// in this case, the write operation has to be one to one relation with the fields in the read operation,
// since a write operation can only take a list of input fields that come from the previous stage
Operation origin = operationsMap.get(input.getOrigin());
if (origin.getType() == OperationType.READ) {
fields.add(new EndPointField(((ReadOperation) origin).getSource(), input.getName()));
continue;
}
fields.addAll(computeIncomingSummaryHelper(origin, write, operationEndPointMap));
}
}
}
for (TransformOperation transform : dropTransforms) {
for (InputField input : transform.getInputs()) {
Operation previous = operationsMap.get(input.getOrigin());
// drop transforms uses a common NULL endpoint as key
Set<EndPointField> endPointFields = summary.computeIfAbsent(NULL_EPF, k -> new HashSet<>());
if (operationEndPointMap.containsKey(input.getOrigin())) {
endPointFields.addAll(new HashSet<>(operationEndPointMap.get(input.getOrigin())));
continue;
}
endPointFields.addAll(computeIncomingSummaryHelper(previous, transform, operationEndPointMap));
}
}
return summary;
}
Aggregations