Examples with Connection io.cdap.cdap.etl.proto.Connection used on opensource projects

Example 91 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ConnectorDagTest method testSplitDag.

@Test
public void testSplitDag() {
    /*
             |--- n2(r) ----------|
             |                    |                                    |-- n10
        n1 --|--- n3(r) --- n5 ---|--- n6 --- n7(r) --- n8 --- n9(r) --|
             |                    |                                    |-- n11
             |--- n4(r) ----------|

        There should be a connector after n1, before n7, and before n9. This should result in subdags:

        n1 --> n1.out.connector

        n1.out.connector --> n2(r) --> n6 --> n7.connector

        n1.out.connector --> n3(r) --> n5 --> n6 --> n7.connector

        n1.out.connector --> n4(r) --> n6 --> n7.connector

        n7.connector --> n7 --> n8 --> n9.connector

                              |--> n10
        n9.connector --> n9 --|
                              |--> n11
     */
    ConnectorDag cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n1", "n4").addConnection("n2", "n6").addConnection("n3", "n5").addConnection("n4", "n6").addConnection("n5", "n6").addConnection("n6", "n7").addConnection("n7", "n8").addConnection("n8", "n9").addConnection("n9", "n10").addConnection("n9", "n11").addReduceNodes("n2", "n3", "n4", "n7", "n9").build();
    cdag.insertConnectors();
    Set<Dag> actual = new HashSet<>(cdag.split());
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n1.out.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n2"), new Connection("n2", "n6"), new Connection("n6", "n7.connector")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n3"), new Connection("n3", "n5"), new Connection("n5", "n6"), new Connection("n6", "n7.connector")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n4"), new Connection("n4", "n6"), new Connection("n6", "n7.connector")));
    Dag dag5 = new Dag(ImmutableSet.of(new Connection("n7.connector", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9.connector")));
    Dag dag6 = new Dag(ImmutableSet.of(new Connection("n9.connector", "n9"), new Connection("n9", "n10"), new Connection("n9", "n11")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4, dag5, dag6);
    Assert.assertEquals(expected, actual);
    /*
             |---> n2(r)
             |      |
        n1 --|      |
             |      v
             |---> n3(r) ---> n4

        n2 and n3 should have connectors inserted in front of them to become:

             |---> n2.connector ---> n2(r)
             |                        |
        n1 --|                        |
             |                        v
             |-------------------> n3.connector ---> n3(r) ---> n4
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n3").addConnection("n3", "n4").addReduceNodes("n2", "n3").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector
        n1 --|
             |--> n3.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3.connector")));
    /*
        n2.connector --> n2 --> n3.connector
     */
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n3.connector")));
    /*
        n3.connector --> n3 --> n4
     */
    dag3 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3"), new Connection("n3", "n4")));
    expected = ImmutableSet.of(dag1, dag2, dag3);
    Assert.assertEquals(expected, actual);
    /*
         n1 --> n2 --|
                     |--> n3(r) --> n4 --|
         n7 --> n8 --|                   |--> n5(r) --> n6
                                         |
         n9 -----------------------------|

        only n5 should have a connector inserted in front of it to become:

         n1 --> n2 --|
                     |--> n3(r) --> n4 --|
         n7 --> n8 --|                   |--> n5.connector --> n5(r) --> n6
                                         |
         n9 -----------------------------|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n2", "n3").addConnection("n3", "n4").addConnection("n4", "n5").addConnection("n5", "n6").addConnection("n7", "n8").addConnection("n8", "n3").addConnection("n9", "n5").addReduceNodes("n3", "n5").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
         n1 --> n2 --|
                     |--> n3(r) --> n4 --|
         n7 --> n8 --|                   |--> n5.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5.connector"), new Connection("n7", "n8"), new Connection("n8", "n3")));
    /*
                                         |--> n5.connector
                                         |
         n9 -----------------------------|
     */
    dag2 = new Dag(ImmutableSet.of(new Connection("n9", "n5.connector")));
    /*
         n5.connector --> n5(r) --> n6
     */
    dag3 = new Dag(ImmutableSet.of(new Connection("n5.connector", "n5"), new Connection("n5", "n6")));
    expected = ImmutableSet.of(dag1, dag2, dag3);
    Assert.assertEquals(expected, actual);
}

Example 92 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ConnectorDagTest method testMultipleNonNestedConditions.

@Test
public void testMultipleNonNestedConditions() {
    /*
       n1-c1-n2-n3-c2-n4
     */
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "c1"), new Connection("c1", "n2"), new Connection("n2", "n3"), new Connection("n3", "c2"), new Connection("c2", "n4"));
    Set<String> conditions = new HashSet<>(Arrays.asList("c1", "c2"));
    Set<String> reduceNodes = new HashSet<>();
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "c1")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("c1", "n2"), new Connection("n2", "n3"), new Connection("n3", "c2")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("c2", "n4")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3);
    Assert.assertEquals(actual, expected);
}

Example 93 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ControlDagTest method testNoOpTrim.

@Test
public void testNoOpTrim() {
    /*
                           |--> n4
        n1 --> n2 --> n3 --|
                           |--> n5
     */
    ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n3", "n4"), new Connection("n3", "n5")));
    Assert.assertEquals(0, cdag.trim());
    /*
             |--> n2 --|
        n1 --|         |--> n4
             |--> n3 --|
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n2", "n4"), new Connection("n3", "n4")));
    Assert.assertEquals(0, cdag.trim());
    /*
                |--> n2 --|
             |--|         |
             |  |--> n3 --|--> n5
        n1 --|            |
             |-----> n4---|
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n5")));
    Assert.assertEquals(0, cdag.trim());
    /*
              |--> n2 --|
              |         |--> n5
         n1 --|--> n3 --|
              |
              |--> n4 --> n6
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n6")));
    Assert.assertEquals(0, cdag.trim());
}

Example 94 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ControlDagTest method testFlatten.

@Test
public void testFlatten() {
    /*
                      |--> n3
            |--> n2 --|
            |         |--> n4
       n1 --|
            |
            |--> n5
     */
    ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4")));
    cdag.flatten();
    /*
            |--> n2 --|             |--> n3 --|
       n1 --|         |--> n2.n5 -->|         |--> n3.n4
            |--> n5 --|             |--> n4 --|
     */
    ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n3", "n3.n4"), new Connection("n4", "n3.n4")));
    Assert.assertEquals(expected, cdag);
    /*
                      |--> n3
            |--> n2 --|
            |         |--> n4
       n1 --|              |
            |              v
            |--> n5 -----> n6
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n4", "n6"), new Connection("n5", "n6")));
    cdag.flatten();
    /*


            |--> n2 --|
            |         |            |--> n3 ---------|
       n1 --|         |--> n2.n5 --|                |--> n3.n6
            |         |            |--> n4 --> n6 --|
            |--> n5 --|
     */
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n4", "n6"), new Connection("n3", "n3.n6"), new Connection("n6", "n3.n6")));
    Assert.assertEquals(expected, cdag);
    /*
              |--> n2 --|
              |         |--> n5
         n1 --|--> n3 --|
              |
              |--> n4 --> n6
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n6")));
    cdag.flatten();
    /*
              |--> n2 ---------|
              |                |
         n1 --|--> n3 ---------|--> n2.n3.n6 --> n5
              |                |
              |--> n4 --> n6 --|
     */
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n4", "n6"), new Connection("n2", "n2.n3.n6"), new Connection("n3", "n2.n3.n6"), new Connection("n6", "n2.n3.n6"), new Connection("n2.n3.n6", "n5")));
    Assert.assertEquals(expected, cdag);
}

Example 95 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ControlDagTest method testTrim.

@Test
public void testTrim() {
    /*
             |--> n2 --|
        n1 --|         v
             |-------> n3
     */
    ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n2", "n3")));
    /*
        trims down to:
        n1 --> n2 --> n3
     */
    Assert.assertEquals(1, cdag.trim());
    ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3")));
    Assert.assertEquals(expected, cdag);
    /*
                       |--> n3 --|
             |--> n2 --|         |--> n5
             |         |--> n4 --|    ^
        n1 --|              |         |
             |              v         |
             |--> n6 -----> n7 -------|
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n3", "n5"), new Connection("n4", "n5"), new Connection("n4", "n7"), new Connection("n6", "n7"), new Connection("n7", "n5")));
    /*
       trims down to:
                       |--> n3 --> n5
             |--> n2 --|           ^
             |         |--> n4     |
        n1 --|              |      |
             |              v      |
             |--> n6 -----> n7 ----|
     */
    Assert.assertEquals(1, cdag.trim());
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n3", "n5"), new Connection("n4", "n7"), new Connection("n6", "n7"), new Connection("n7", "n5")));
    Assert.assertEquals(expected, cdag);
    /*
             |--> n2 --> n3 ------------
        n1 --|           |      |      |
             |           v      v      |
             |---------> n4 --> n5 ----|
             |           |             v
             |-----------------------> n6
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n4"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n3", "n5"), new Connection("n3", "n6"), new Connection("n4", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6")));
    /*
       trims down to:
        n1 --> n2 --> n3 --> n4 --> n5 --> n6
     */
    Assert.assertEquals(5, cdag.trim());
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6")));
    Assert.assertEquals(expected, cdag);
}