Examples with CollapsedRelation co.cask.cdap.data2.metadata.lineage.CollapsedRelation used on opensource projects

Example 1 with CollapsedRelation

use of co.cask.cdap.data2.metadata.lineage.CollapsedRelation in project cdap by caskdata.

the class LineageCollapserTest method testCollapseComponent.

@Test
public void testCollapseComponent() throws Exception {
    Set<Relation> relations = ImmutableSet.of(new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.WRITE, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet12)));
    // Collapse on component
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ), toSet(runId1), toSet(flowlet11, flowlet12)), new CollapsedRelation(data1, flow1, toSet(AccessType.WRITE), toSet(runId1), toSet(flowlet11))), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.COMPONENT)));
}

Example 2 with CollapsedRelation

use of co.cask.cdap.data2.metadata.lineage.CollapsedRelation in project cdap by caskdata.

the class LineageCollapserTest method testCollapseMulti.

@Test
public void testCollapseMulti() throws Exception {
    Set<Relation> relations = ImmutableSet.of(new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.WRITE, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet12)), new Relation(data1, flow2, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow2, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data2, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data2, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)));
    // Collapse on access
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1), toSet(flowlet11)), new CollapsedRelation(data1, flow1, toSet(AccessType.READ), toSet(runId1), toSet(flowlet12)), new CollapsedRelation(data1, flow2, toSet(AccessType.READ), toSet(runId1), toSet(flowlet11)), new CollapsedRelation(data2, flow1, toSet(AccessType.READ), toSet(runId1), toSet(flowlet11))), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.ACCESS)));
}

Example 3 with CollapsedRelation

use of co.cask.cdap.data2.metadata.lineage.CollapsedRelation in project cdap by caskdata.

the class LineageCollapserTest method testCollapseAccess.

@Test
public void testCollapseAccess() throws Exception {
    Set<Relation> relations = ImmutableSet.of(new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.WRITE, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet12)));
    // Collapse on access
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1), toSet(flowlet11)), new CollapsedRelation(data1, flow1, toSet(AccessType.READ), toSet(runId1), toSet(flowlet12))), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.ACCESS)));
}

Example 4 with CollapsedRelation

use of co.cask.cdap.data2.metadata.lineage.CollapsedRelation in project cdap by caskdata.

the class LineageCollapserTest method testCollapseCombinations.

@Test
public void testCollapseCombinations() throws Exception {
    Set<Relation> relations = ImmutableSet.of(// First run
    new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.WRITE, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet12)), // Second run
    new Relation(data1, flow1, AccessType.READ, runId2, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.WRITE, runId2, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.READ, runId2, ImmutableSet.of(flowlet12)), new Relation(data1, flow1, AccessType.UNKNOWN, runId2, ImmutableSet.of(flowlet12)), // Third run
    new Relation(data1, flow1, AccessType.READ, runId3, ImmutableSet.of(flowlet12)), new Relation(data1, flow1, AccessType.UNKNOWN, runId3, ImmutableSet.of(flowlet12)));
    // Collapse on access type, run
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1, runId2), toSet(flowlet11)), new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.UNKNOWN), toSet(runId1, runId2, runId3), toSet(flowlet12))), LineageCollapser.collapseRelations(relations, toSet(CollapseType.ACCESS, CollapseType.RUN)));
    // Collapse on access type, component
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1), toSet(flowlet11, flowlet12)), new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.WRITE, AccessType.UNKNOWN), toSet(runId2), toSet(flowlet11, flowlet12)), new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.UNKNOWN), toSet(runId3), toSet(flowlet12))), LineageCollapser.collapseRelations(relations, toSet(CollapseType.ACCESS, CollapseType.COMPONENT)));
    // Collapse on component, run
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ), toSet(runId1, runId2, runId3), toSet(flowlet11, flowlet12)), new CollapsedRelation(data1, flow1, toSet(AccessType.WRITE), toSet(runId1, runId2), toSet(flowlet11)), new CollapsedRelation(data1, flow1, toSet(AccessType.UNKNOWN), toSet(runId2, runId3), toSet(flowlet12))), LineageCollapser.collapseRelations(relations, toSet(CollapseType.COMPONENT, CollapseType.RUN)));
    // Collapse on all three
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ, AccessType.WRITE, AccessType.UNKNOWN), toSet(runId1, runId2, runId3), toSet(flowlet11, flowlet12))), LineageCollapser.collapseRelations(relations, toSet(CollapseType.COMPONENT, CollapseType.RUN, CollapseType.ACCESS)));
}

Example 5 with CollapsedRelation

use of co.cask.cdap.data2.metadata.lineage.CollapsedRelation in project cdap by caskdata.

the class LineageCollapserTest method testCollapseRun.

@Test
public void testCollapseRun() throws Exception {
    Set<Relation> relations = ImmutableSet.of(new Relation(data1, flow1, AccessType.READ, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.WRITE, runId1, ImmutableSet.of(flowlet11)), new Relation(data1, flow1, AccessType.READ, runId2, ImmutableSet.of(flowlet11)));
    // Collapse on run
    Assert.assertEquals(toSet(new CollapsedRelation(data1, flow1, toSet(AccessType.READ), toSet(runId1, runId2), toSet(flowlet11)), new CollapsedRelation(data1, flow1, toSet(AccessType.WRITE), toSet(runId1), toSet(flowlet11))), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.RUN)));
}