Examples with Connection io.cdap.cdap.etl.proto.Connection used on opensource projects

Example 46 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ConnectorDagTest method testSimpleConditionWithReducers.

@Test
public void testSimpleConditionWithReducers() {
    /*
             |--- n2
        n1 --|
             |--- n3(r) --- n4---condition----n5
                                    |
                                    |---------n6
    */
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition"), new Connection("condition", "n5"), new Connection("condition", "n6"));
    Set<String> conditions = Collections.singleton("condition");
    Set<String> reduceNodes = Collections.singleton("n3");
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("condition", "n5")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("condition", "n6")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4);
    Assert.assertEquals(actual, expected);
}

Example 47 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ConnectorDagTest method testSimpleConditionWithMultipleSources.

@Test
public void testSimpleConditionWithMultipleSources() {
    /*
             |--- n2
        n1 --|
             |--- n3(r) --- n4---condition----n5
                    |                |
        n11---------|                |---------n6
    */
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition"), new Connection("condition", "n5"), new Connection("condition", "n6"), new Connection("n11", "n3"));
    Set<String> conditions = Collections.singleton("condition");
    Set<String> reduceNodes = Collections.singleton("n3");
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3.connector"), new Connection("n11", "n3.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("condition", "n5")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("condition", "n6")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4);
    Assert.assertEquals(actual, expected);
}

Example 48 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ConnectorDagTest method testConditionDag.

@Test
public void testConditionDag() {
    /*
          file - csv - c1 - t1---agg1--agg2---sink1
                        |
                        ----c2 - sink2
                             |
                              ------c3 - sink3

     */
    Set<Connection> connections = ImmutableSet.of(new Connection("file", "csv"), new Connection("csv", "c1"), new Connection("c1", "t1"), new Connection("t1", "agg1"), new Connection("agg1", "agg2"), new Connection("agg2", "sink1"), new Connection("c1", "c2"), new Connection("c2", "sink2"), new Connection("c2", "c3"), new Connection("c3", "sink3"));
    Set<String> conditions = new HashSet<>(Arrays.asList("c1", "c2", "c3"));
    Set<String> reduceNodes = new HashSet<>(Arrays.asList("agg1", "agg2"));
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("file", "csv"), new Connection("csv", "c1")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("c1", "t1"), new Connection("t1", "agg1"), new Connection("agg1", "agg2.connector")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("agg2.connector", "agg2"), new Connection("agg2", "sink1")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("c1", "c2")));
    Dag dag5 = new Dag(ImmutableSet.of(new Connection("c2", "sink2")));
    Dag dag6 = new Dag(ImmutableSet.of(new Connection("c2", "c3")));
    Dag dag7 = new Dag(ImmutableSet.of(new Connection("c3", "sink3")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4, dag5, dag6, dag7);
    Assert.assertEquals(actual, expected);
}

Example 49 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ConnectorDagTest method testMergedReduceBranches.

@Test
public void testMergedReduceBranches() {
    /*
              |--> n2(r) --|
         n1 --|            |--> n4
              |--> n3(r) --|
     */
    ConnectorDag cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n4").addConnection("n3", "n4").addReduceNodes("n2", "n3").build();
    cdag.insertConnectors();
    Set<Dag> actual = new HashSet<>(cdag.split());
    /*
                                  n1.out.connector --> n2(r) --> n4.connector
        n1 --> n1.out.connector                                                 n4.connector --> n4
                                  n1.out.connector --> n3(r) --> n4.connector
     */
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n1.out.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n2"), new Connection("n2", "n4.connector")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n3"), new Connection("n3", "n4.connector")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("n4.connector", "n4")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3, dag4), actual);
    /*
              |-- n2(r) --|
         n1 --|           |-- n4
              |-- n3 -----|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n4").addConnection("n3", "n4").addReduceNodes("n2").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector   n2.connector --> n2(r) --> n4.connector
        n1 --|                                                              n4.connector --> n4
             |--> n3 --> n4.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3"), new Connection("n3", "n4.connector")));
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n4.connector")));
    dag3 = new Dag(ImmutableSet.of(new Connection("n4.connector", "n4")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3), actual);
    /*
              |-- n2(r) --|
         n1 --|           |-- n3
              |-----------|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n3").addReduceNodes("n2").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector   n2.connector --> n2(r) --> n3.connector
        n1 --|                                                              n3.connector --> n3
             |--> n3.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3.connector")));
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n3.connector")));
    dag3 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3), actual);
}

Example 50 with Connection

use of io.cdap.cdap.etl.proto.Connection in project cdap by cdapio.

the class ControlDagTest method testFlattenNoOp.

@Test
public void testFlattenNoOp() {
    /*
        n1 --> n2 --> n3
     */
    ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3")));
    cdag.flatten();
    ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3")));
    Assert.assertEquals(expected, cdag);
    /*
             |--> n2 --|
             |         |         |--> n6 --|
        n1 --|--> n3 --|--> n5 --|         |--> n8
             |         |         |--> n7 --|
             |--> n4 --|
     */
    cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n5", "n7"), new Connection("n6", "n8"), new Connection("n7", "n8")));
    cdag.flatten();
    expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n5", "n7"), new Connection("n6", "n8"), new Connection("n7", "n8")));
    Assert.assertEquals(expected, cdag);
}