use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.
the class ConnectorDagTest method testSplitDag.
@Test
public void testSplitDag() {
/*
|--- n2(r) ----------|
| | |-- n10
n1 --|--- n3(r) --- n5 ---|--- n6 --- n7(r) --- n8 --- n9(r) --|
| | |-- n11
|--- n4(r) ----------|
There should be a connector after n1, before n7, and before n9. This should result in subdags:
n1 --> n1.out.connector
n1.out.connector --> n2(r) --> n6 --> n7.connector
n1.out.connector --> n3(r) --> n5 --> n6 --> n7.connector
n1.out.connector --> n4(r) --> n6 --> n7.connector
n7.connector --> n7 --> n8 --> n9.connector
|--> n10
n9.connector --> n9 --|
|--> n11
*/
ConnectorDag cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n1", "n4").addConnection("n2", "n6").addConnection("n3", "n5").addConnection("n4", "n6").addConnection("n5", "n6").addConnection("n6", "n7").addConnection("n7", "n8").addConnection("n8", "n9").addConnection("n9", "n10").addConnection("n9", "n11").addReduceNodes("n2", "n3", "n4", "n7", "n9").build();
cdag.insertConnectors();
Set<Dag> actual = new HashSet<>(cdag.split());
Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n1.out.connector")));
Dag dag2 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n2"), new Connection("n2", "n6"), new Connection("n6", "n7.connector")));
Dag dag3 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n3"), new Connection("n3", "n5"), new Connection("n5", "n6"), new Connection("n6", "n7.connector")));
Dag dag4 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n4"), new Connection("n4", "n6"), new Connection("n6", "n7.connector")));
Dag dag5 = new Dag(ImmutableSet.of(new Connection("n7.connector", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9.connector")));
Dag dag6 = new Dag(ImmutableSet.of(new Connection("n9.connector", "n9"), new Connection("n9", "n10"), new Connection("n9", "n11")));
Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4, dag5, dag6);
Assert.assertEquals(expected, actual);
/*
|---> n2(r)
| |
n1 --| |
| v
|---> n3(r) ---> n4
n2 and n3 should have connectors inserted in front of them to become:
|---> n2.connector ---> n2(r)
| |
n1 --| |
| v
|-------------------> n3.connector ---> n3(r) ---> n4
*/
cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n3").addConnection("n3", "n4").addReduceNodes("n2", "n3").build();
cdag.insertConnectors();
actual = new HashSet<>(cdag.split());
/*
|--> n2.connector
n1 --|
|--> n3.connector
*/
dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3.connector")));
/*
n2.connector --> n2 --> n3.connector
*/
dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n3.connector")));
/*
n3.connector --> n3 --> n4
*/
dag3 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3"), new Connection("n3", "n4")));
expected = ImmutableSet.of(dag1, dag2, dag3);
Assert.assertEquals(expected, actual);
/*
n1 --> n2 --|
|--> n3(r) --> n4 --|
n7 --> n8 --| |--> n5(r) --> n6
|
n9 -----------------------------|
only n5 should have a connector inserted in front of it to become:
n1 --> n2 --|
|--> n3(r) --> n4 --|
n7 --> n8 --| |--> n5.connector --> n5(r) --> n6
|
n9 -----------------------------|
*/
cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n2", "n3").addConnection("n3", "n4").addConnection("n4", "n5").addConnection("n5", "n6").addConnection("n7", "n8").addConnection("n8", "n3").addConnection("n9", "n5").addReduceNodes("n3", "n5").build();
cdag.insertConnectors();
actual = new HashSet<>(cdag.split());
/*
n1 --> n2 --|
|--> n3(r) --> n4 --|
n7 --> n8 --| |--> n5.connector
*/
dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5.connector"), new Connection("n7", "n8"), new Connection("n8", "n3")));
/*
|--> n5.connector
|
n9 -----------------------------|
*/
dag2 = new Dag(ImmutableSet.of(new Connection("n9", "n5.connector")));
/*
n5.connector --> n5(r) --> n6
*/
dag3 = new Dag(ImmutableSet.of(new Connection("n5.connector", "n5"), new Connection("n5", "n6")));
expected = ImmutableSet.of(dag1, dag2, dag3);
Assert.assertEquals(expected, actual);
}
use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.
the class ConnectorDagTest method testMultipleNonNestedConditions.
@Test
public void testMultipleNonNestedConditions() {
/*
n1-c1-n2-n3-c2-n4
*/
Set<Connection> connections = ImmutableSet.of(new Connection("n1", "c1"), new Connection("c1", "n2"), new Connection("n2", "n3"), new Connection("n3", "c2"), new Connection("c2", "n4"));
Set<String> conditions = new HashSet<>(Arrays.asList("c1", "c2"));
Set<String> reduceNodes = new HashSet<>();
Set<String> isolationNodes = new HashSet<>();
Set<String> multiPortNodes = new HashSet<>();
Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "c1")));
Dag dag2 = new Dag(ImmutableSet.of(new Connection("c1", "n2"), new Connection("n2", "n3"), new Connection("n3", "c2")));
Dag dag3 = new Dag(ImmutableSet.of(new Connection("c2", "n4")));
Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3);
Assert.assertEquals(actual, expected);
}
use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.
the class ControlDagTest method testNoOpTrim.
@Test
public void testNoOpTrim() {
/*
|--> n4
n1 --> n2 --> n3 --|
|--> n5
*/
ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n3", "n4"), new Connection("n3", "n5")));
Assert.assertEquals(0, cdag.trim());
/*
|--> n2 --|
n1 --| |--> n4
|--> n3 --|
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n2", "n4"), new Connection("n3", "n4")));
Assert.assertEquals(0, cdag.trim());
/*
|--> n2 --|
|--| |
| |--> n3 --|--> n5
n1 --| |
|-----> n4---|
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n5")));
Assert.assertEquals(0, cdag.trim());
/*
|--> n2 --|
| |--> n5
n1 --|--> n3 --|
|
|--> n4 --> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n6")));
Assert.assertEquals(0, cdag.trim());
}
use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.
the class ControlDagTest method testFlatten.
@Test
public void testFlatten() {
/*
|--> n3
|--> n2 --|
| |--> n4
n1 --|
|
|--> n5
*/
ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4")));
cdag.flatten();
/*
|--> n2 --| |--> n3 --|
n1 --| |--> n2.n5 -->| |--> n3.n4
|--> n5 --| |--> n4 --|
*/
ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n3", "n3.n4"), new Connection("n4", "n3.n4")));
Assert.assertEquals(expected, cdag);
/*
|--> n3
|--> n2 --|
| |--> n4
n1 --| |
| v
|--> n5 -----> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n4", "n6"), new Connection("n5", "n6")));
cdag.flatten();
/*
|--> n2 --|
| | |--> n3 ---------|
n1 --| |--> n2.n5 --| |--> n3.n6
| | |--> n4 --> n6 --|
|--> n5 --|
*/
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n4", "n6"), new Connection("n3", "n3.n6"), new Connection("n6", "n3.n6")));
Assert.assertEquals(expected, cdag);
/*
|--> n2 --|
| |--> n5
n1 --|--> n3 --|
|
|--> n4 --> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n6")));
cdag.flatten();
/*
|--> n2 ---------|
| |
n1 --|--> n3 ---------|--> n2.n3.n6 --> n5
| |
|--> n4 --> n6 --|
*/
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n4", "n6"), new Connection("n2", "n2.n3.n6"), new Connection("n3", "n2.n3.n6"), new Connection("n6", "n2.n3.n6"), new Connection("n2.n3.n6", "n5")));
Assert.assertEquals(expected, cdag);
}
use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.
the class ControlDagTest method testTrim.
@Test
public void testTrim() {
/*
|--> n2 --|
n1 --| v
|-------> n3
*/
ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n2", "n3")));
/*
trims down to:
n1 --> n2 --> n3
*/
Assert.assertEquals(1, cdag.trim());
ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3")));
Assert.assertEquals(expected, cdag);
/*
|--> n3 --|
|--> n2 --| |--> n5
| |--> n4 --| ^
n1 --| | |
| v |
|--> n6 -----> n7 -------|
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n3", "n5"), new Connection("n4", "n5"), new Connection("n4", "n7"), new Connection("n6", "n7"), new Connection("n7", "n5")));
/*
trims down to:
|--> n3 --> n5
|--> n2 --| ^
| |--> n4 |
n1 --| | |
| v |
|--> n6 -----> n7 ----|
*/
Assert.assertEquals(1, cdag.trim());
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n3", "n5"), new Connection("n4", "n7"), new Connection("n6", "n7"), new Connection("n7", "n5")));
Assert.assertEquals(expected, cdag);
/*
|--> n2 --> n3 ------------
n1 --| | | |
| v v |
|---------> n4 --> n5 ----|
| | v
|-----------------------> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n4"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n3", "n5"), new Connection("n3", "n6"), new Connection("n4", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6")));
/*
trims down to:
n1 --> n2 --> n3 --> n4 --> n5 --> n6
*/
Assert.assertEquals(5, cdag.trim());
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6")));
Assert.assertEquals(expected, cdag);
}
Aggregations