Examples with Connection io.cdap.cdap.etl.proto.Connection used on opensource projects

Example 11 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by caskdata.

the class ConnectorDagTest method testMergedReduceBranches.

@Test
public void testMergedReduceBranches() {
    /*
              |--> n2(r) --|
         n1 --|            |--> n4
              |--> n3(r) --|
     */
    ConnectorDag cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n4").addConnection("n3", "n4").addReduceNodes("n2", "n3").build();
    cdag.insertConnectors();
    Set<Dag> actual = new HashSet<>(cdag.split());
    /*
                                  n1.out.connector --> n2(r) --> n4.connector
        n1 --> n1.out.connector                                                 n4.connector --> n4
                                  n1.out.connector --> n3(r) --> n4.connector
     */
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n1.out.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n2"), new Connection("n2", "n4.connector")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n3"), new Connection("n3", "n4.connector")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("n4.connector", "n4")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3, dag4), actual);
    /*
              |-- n2(r) --|
         n1 --|           |-- n4
              |-- n3 -----|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n4").addConnection("n3", "n4").addReduceNodes("n2").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector   n2.connector --> n2(r) --> n4.connector
        n1 --|                                                              n4.connector --> n4
             |--> n3 --> n4.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3"), new Connection("n3", "n4.connector")));
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n4.connector")));
    dag3 = new Dag(ImmutableSet.of(new Connection("n4.connector", "n4")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3), actual);
    /*
              |-- n2(r) --|
         n1 --|           |-- n3
              |-----------|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n3").addReduceNodes("n2").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector   n2.connector --> n2(r) --> n3.connector
        n1 --|                                                              n3.connector --> n3
             |--> n3.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3.connector")));
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n3.connector")));
    dag3 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3), actual);
}

Example 12 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by caskdata.

the class DagTest method testSubsetAround.

@Test
public void testSubsetAround() {
    /*
         n1 --> n2 --|
                     |--> n3 --> n4 --|
         n7 --> n8 --|                |--> n5 --> n6
                                      |
         n9 --------------------------|
     */
    Dag fullDag = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n7", "n8"), new Connection("n8", "n3"), new Connection("n9", "n5")));
    // test without stop nodes
    /*
         n1 --> n2 --|
                     |--> n3 --> n4 --|
                                      |--> n5 --> n6
     */
    Dag expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6")));
    Assert.assertEquals(expected, fullDag.subsetAround("n2", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    Assert.assertEquals(expected, fullDag.subsetAround("n1", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    /*
         n1 --> n2 --|
                     |--> n3 --> n4 --|
         n7 --> n8 --|                |--> n5 --> n6
     */
    expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n7", "n8"), new Connection("n8", "n3")));
    Assert.assertEquals(expected, fullDag.subsetAround("n3", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    Assert.assertEquals(expected, fullDag.subsetAround("n4", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    Assert.assertEquals(fullDag, fullDag.subsetAround("n5", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    Assert.assertEquals(fullDag, fullDag.subsetAround("n6", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    /*
                     |--> n3 --> n4 --|
         n7 --> n8 --|                |--> n5 --> n6
     */
    expected = new Dag(ImmutableSet.of(new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n7", "n8"), new Connection("n8", "n3")));
    Assert.assertEquals(expected, fullDag.subsetAround("n7", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    Assert.assertEquals(expected, fullDag.subsetAround("n8", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    /*
                                      |--> n5 --> n6
                                      |
         n9 --------------------------|
     */
    expected = new Dag(ImmutableSet.of(new Connection("n5", "n6"), new Connection("n9", "n5")));
    Assert.assertEquals(expected, fullDag.subsetAround("n9", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
    // test with stop nodes
    /*
         n1 --> n2 --|
                     |--> n3 --> n4
                n8 --|
     */
    expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n8", "n3")));
    Assert.assertEquals(expected, fullDag.subsetAround("n3", ImmutableSet.of("n4"), ImmutableSet.of("n1", "n8")));
    /*
         n1 --> n2 --|
                     |--> n3 --> n4
     */
    expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4")));
    Assert.assertEquals(expected, fullDag.subsetAround("n2", ImmutableSet.of("n4"), ImmutableSet.of("n1", "n8")));
    Assert.assertEquals(expected, fullDag.subsetAround("n1", ImmutableSet.of("n4"), ImmutableSet.of("n1", "n8")));
    /*
                          n3 --> n4 --|
                                      |--> n5 --> n6
                                      |
         n9 --------------------------|
     */
    expected = new Dag(ImmutableSet.of(new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n9", "n5")));
    Assert.assertEquals(expected, fullDag.subsetAround("n5", ImmutableSet.of("n6"), ImmutableSet.of("n3", "n9")));
    Assert.assertEquals(expected, fullDag.subsetAround("n6", ImmutableSet.of("n6"), ImmutableSet.of("n3", "n9")));
    /*
                n2 --|
                     |--> n3 --> n4 --|
                n8 --|                |--> n5
                                      |
         n9 --------------------------|
     */
    expected = new Dag(ImmutableSet.of(new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n8", "n3"), new Connection("n9", "n5")));
    Assert.assertEquals(expected, fullDag.subsetAround("n5", ImmutableSet.of("n6"), ImmutableSet.of("n2", "n8")));
    /*
                n2 --|
                     |--> n3 --> n4 --|
                n8 --|                |--> n5
     */
    expected = new Dag(ImmutableSet.of(new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n8", "n3")));
    Assert.assertEquals(expected, fullDag.subsetAround("n4", ImmutableSet.of("n5"), ImmutableSet.of("n2", "n8")));
}

Example 13 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by caskdata.

the class DagTest method testSubset.

@Test
public void testSubset() {
    /*
        n1 -- n2
              |
              v
        n3 -- n4 --- n8
              ^
              |
        n5-------- n6 -- n7
     */
    Dag fulldag = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n4"), new Connection("n3", "n4"), new Connection("n4", "n8"), new Connection("n5", "n4"), new Connection("n5", "n6"), new Connection("n6", "n7")));
    Dag expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n4"), new Connection("n4", "n8")));
    Dag actual = fulldag.subsetFrom("n1");
    Assert.assertEquals(expected, actual);
    expected = new Dag(ImmutableSet.of(new Connection("n2", "n4"), new Connection("n4", "n8")));
    actual = fulldag.subsetFrom("n2");
    Assert.assertEquals(expected, actual);
    expected = new Dag(ImmutableSet.of(new Connection("n3", "n4"), new Connection("n4", "n8")));
    actual = fulldag.subsetFrom("n3");
    Assert.assertEquals(expected, actual);
    expected = new Dag(ImmutableSet.of(new Connection("n4", "n8"), new Connection("n5", "n4"), new Connection("n5", "n6"), new Connection("n6", "n7")));
    actual = fulldag.subsetFrom("n5");
    Assert.assertEquals(expected, actual);
    expected = new Dag(ImmutableSet.of(new Connection("n6", "n7")));
    actual = fulldag.subsetFrom("n6");
    Assert.assertEquals(expected, actual);
    // test subsets with stop nodes
    expected = new Dag(ImmutableSet.of(new Connection("n1", "n2")));
    actual = fulldag.subsetFrom("n1", ImmutableSet.of("n2"));
    Assert.assertEquals(expected, actual);
    expected = new Dag(ImmutableSet.of(new Connection("n5", "n4"), new Connection("n5", "n6")));
    actual = fulldag.subsetFrom("n5", ImmutableSet.of("n4", "n6"));
    Assert.assertEquals(expected, actual);
    /*
             |--- n2 ----------|
             |                 |                              |-- n10
        n1 --|--- n3 --- n5 ---|--- n6 --- n7 --- n8 --- n9 --|
             |                 |                              |-- n11
             |--- n4 ----------|

     */
    fulldag = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n6"), new Connection("n3", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9"), new Connection("n9", "n10"), new Connection("n9", "n11")));
    expected = new Dag(ImmutableSet.of(new Connection("n3", "n5"), new Connection("n5", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9")));
    actual = fulldag.subsetFrom("n3", ImmutableSet.of("n4", "n9"));
    Assert.assertEquals(expected, actual);
    expected = new Dag(ImmutableSet.of(new Connection("n2", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8")));
    actual = fulldag.subsetFrom("n2", ImmutableSet.of("n4", "n8", "n1"));
    Assert.assertEquals(expected, actual);
}

Example 14 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by caskdata.

the class DagTest method testComplicatedSplitByControl.

@Test
public void testComplicatedSplitByControl() {
    /*
                                                   |-- n2 -- a3
            |-- a1 --|        |-- n0 -- n1 -- c1 --|                          |-- a5 --|
        a0--|        |-- c0 --|                    |-- n3 -- c2 -- n8 -- a4 --|        |-- a7
            |-- a2 --|        |                                               |-- a6 --|
                              |        |-- n4 -- n5 -- c4 -- c5 -- n9
                              |-- c3 --|
                                       |              |-- a8
                                       |-- n6 -- n7 --|
                                                      |-- a9
     */
    Dag dag = new Dag(ImmutableSet.of(new Connection("a0", "a1"), new Connection("a0", "a2"), new Connection("a1", "c0"), new Connection("a2", "c0"), new Connection("c0", "n0"), new Connection("c0", "c3"), new Connection("n0", "n1"), new Connection("n1", "c1"), new Connection("c1", "n2"), new Connection("c1", "n3"), new Connection("n2", "a3"), new Connection("n3", "c2"), new Connection("c2", "n8"), new Connection("n8", "a4"), new Connection("a4", "a5"), new Connection("a4", "a6"), new Connection("a5", "a7"), new Connection("a6", "a7"), new Connection("c3", "n4"), new Connection("c3", "n6"), new Connection("n4", "n5"), new Connection("n5", "c4"), new Connection("c4", "c5"), new Connection("c5", "n9"), new Connection("n6", "n7"), new Connection("n7", "a8"), new Connection("n7", "a9")));
    Set<Dag> actual = dag.splitByControlNodes(ImmutableSet.of("c0", "c1", "c2", "c3", "c4", "c5"), ImmutableSet.of("a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9"));
    Set<Dag> expected = ImmutableSet.of(new Dag(ImmutableSet.of(new Connection("a0", "a1"), new Connection("a0", "a2"))), new Dag(ImmutableSet.of(new Connection("a1", "c0"), new Connection("a2", "c0"))), new Dag(ImmutableSet.of(new Connection("c0", "n0"), new Connection("n0", "n1"), new Connection("n1", "c1"))), new Dag(ImmutableSet.of(new Connection("c0", "c3"))), new Dag(ImmutableSet.of(new Connection("c1", "n2"), new Connection("n2", "a3"))), new Dag(ImmutableSet.of(new Connection("c1", "n3"), new Connection("n3", "c2"))), new Dag(ImmutableSet.of(new Connection("c2", "n8"), new Connection("n8", "a4"))), new Dag(ImmutableSet.of(new Connection("a4", "a5"), new Connection("a4", "a6"))), new Dag(ImmutableSet.of(new Connection("a5", "a7"), new Connection("a6", "a7"))), new Dag(ImmutableSet.of(new Connection("c3", "n4"), new Connection("n4", "n5"), new Connection("n5", "c4"))), new Dag(ImmutableSet.of(new Connection("c3", "n6"), new Connection("n6", "n7"), new Connection("n7", "a8"), new Connection("n7", "a9"))), new Dag(ImmutableSet.of(new Connection("c4", "c5"))), new Dag(ImmutableSet.of(new Connection("c5", "n9"))));
    Assert.assertEquals(expected, actual);
}

Example 15 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by caskdata.

the class ControlDagTest method testFlatten.

@Test
public void testFlatten() {
    /*
                      |--> n3
            |--> n2 --|
            |         |--> n4
       n1 --|
            |
            |--> n5
     */
    ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4")));
    cdag.flatten();
    /*
            |--> n2 --|             |--> n3 --|
       n1 --|         |--> n2.n5 -->|         |--> n3.n4
            |--> n5 --|             |--> n4 --|
     */
    ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n3", "n3.n4"), new Connection("n4", "n3.n4")));
    Assert.assertEquals(expected, cdag);
    /*
                      |--> n3
            |--> n2 --|
            |         |--> n4
       n1 --|              |
            |              v
            |--> n5 -----> n6
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n4", "n6"), new Connection("n5", "n6")));
    cdag.flatten();
    /*


            |--> n2 --|
            |         |            |--> n3 ---------|
       n1 --|         |--> n2.n5 --|                |--> n3.n6
            |         |            |--> n4 --> n6 --|
            |--> n5 --|
     */
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n4", "n6"), new Connection("n3", "n3.n6"), new Connection("n6", "n3.n6")));
    Assert.assertEquals(expected, cdag);
    /*
              |--> n2 --|
              |         |--> n5
         n1 --|--> n3 --|
              |
              |--> n4 --> n6
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n6")));
    cdag.flatten();
    /*
              |--> n2 ---------|
              |                |
         n1 --|--> n3 ---------|--> n2.n3.n6 --> n5
              |                |
              |--> n4 --> n6 --|
     */
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n4", "n6"), new Connection("n2", "n2.n3.n6"), new Connection("n3", "n2.n3.n6"), new Connection("n6", "n2.n3.n6"), new Connection("n2.n3.n6", "n5")));
    Assert.assertEquals(expected, cdag);
}