Examples with Connection co.cask.cdap.etl.proto.Connection used on opensource projects

Example 21 with Connection

use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.

the class ConnectorDagTest method testSimpleConditionWithMultipleSources.

@Test
public void testSimpleConditionWithMultipleSources() throws Exception {
    /*
             |--- n2
        n1 --|
             |--- n3(r) --- n4---condition----n5
                    |                |
        n11---------|                |---------n6
    */
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition"), new Connection("condition", "n5"), new Connection("condition", "n6"), new Connection("n11", "n3"));
    Set<String> conditions = new HashSet<>(Arrays.asList("condition"));
    Set<String> reduceNodes = new HashSet<>(Arrays.asList("n3"));
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3.connector"), new Connection("n11", "n3.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("condition", "n5")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("condition", "n6")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4);
    Assert.assertEquals(actual, expected);
}

Example 22 with Connection

use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.

the class ConnectorDagTest method testSimpleCondition.

@Test
public void testSimpleCondition() throws Exception {
    /*
      file - csv - condition - sink1
                      |
                      |-------sink2
     */
    Set<Connection> connections = ImmutableSet.of(new Connection("file", "csv"), new Connection("csv", "condition"), new Connection("condition", "sink1"), new Connection("condition", "sink2"));
    Set<String> conditions = new HashSet<>(Arrays.asList("condition"));
    Set<String> reduceNodes = new HashSet<>();
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("file", "csv"), new Connection("csv", "condition")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("condition", "sink1")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("condition", "sink2")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3);
    Assert.assertEquals(actual, expected);
}

Example 23 with Connection

use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.

the class ConnectorDagTest method testMergedReduceBranches.

@Test
public void testMergedReduceBranches() {
    /*
              |--> n2(r) --|
         n1 --|            |--> n4
              |--> n3(r) --|
     */
    ConnectorDag cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n4").addConnection("n3", "n4").addReduceNodes("n2", "n3").build();
    cdag.insertConnectors();
    Set<Dag> actual = new HashSet<>(cdag.split());
    /*
                                  n1.out.connector --> n2(r) --> n4.connector
        n1 --> n1.out.connector                                                 n4.connector --> n4
                                  n1.out.connector --> n3(r) --> n4.connector
     */
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n1.out.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n2"), new Connection("n2", "n4.connector")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("n1.out.connector", "n3"), new Connection("n3", "n4.connector")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("n4.connector", "n4")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3, dag4), actual);
    /*
              |-- n2(r) --|
         n1 --|           |-- n4
              |-- n3 -----|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n4").addConnection("n3", "n4").addReduceNodes("n2").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector   n2.connector --> n2(r) --> n4.connector
        n1 --|                                                              n4.connector --> n4
             |--> n3 --> n4.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3"), new Connection("n3", "n4.connector")));
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n4.connector")));
    dag3 = new Dag(ImmutableSet.of(new Connection("n4.connector", "n4")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3), actual);
    /*
              |-- n2(r) --|
         n1 --|           |-- n3
              |-----------|
     */
    cdag = ConnectorDag.builder().addConnection("n1", "n2").addConnection("n1", "n3").addConnection("n2", "n3").addReduceNodes("n2").build();
    cdag.insertConnectors();
    actual = new HashSet<>(cdag.split());
    /*
             |--> n2.connector   n2.connector --> n2(r) --> n3.connector
        n1 --|                                                              n3.connector --> n3
             |--> n3.connector
     */
    dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2.connector"), new Connection("n1", "n3.connector")));
    dag2 = new Dag(ImmutableSet.of(new Connection("n2.connector", "n2"), new Connection("n2", "n3.connector")));
    dag3 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3")));
    Assert.assertEquals(ImmutableSet.of(dag1, dag2, dag3), actual);
}

Example 24 with Connection

use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.

the class ConnectorDagTest method testMultipleNonNestedConditions.

@Test
public void testMultipleNonNestedConditions() throws Exception {
    /*
       n1-c1-n2-n3-c2-n4
     */
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "c1"), new Connection("c1", "n2"), new Connection("n2", "n3"), new Connection("n3", "c2"), new Connection("c2", "n4"));
    Set<String> conditions = new HashSet<>(Arrays.asList("c1", "c2"));
    Set<String> reduceNodes = new HashSet<>();
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "c1")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("c1", "n2"), new Connection("n2", "n3"), new Connection("n3", "c2")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("c2", "n4")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3);
    Assert.assertEquals(actual, expected);
}

Example 25 with Connection

use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.

the class ConnectorDagTest method testSimpleConditionWithReducers.

@Test
public void testSimpleConditionWithReducers() throws Exception {
    /*
             |--- n2
        n1 --|
             |--- n3(r) --- n4---condition----n5
                                    |
                                    |---------n6
    */
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition"), new Connection("condition", "n5"), new Connection("condition", "n6"));
    Set<String> conditions = new HashSet<>(Arrays.asList("condition"));
    Set<String> reduceNodes = new HashSet<>(Arrays.asList("n3"));
    Set<String> isolationNodes = new HashSet<>();
    Set<String> multiPortNodes = new HashSet<>();
    Set<Dag> actual = PipelinePlanner.split(connections, conditions, reduceNodes, isolationNodes, EMPTY_ACTIONS, multiPortNodes, EMPTY_CONNECTORS);
    Dag dag1 = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3.connector")));
    Dag dag2 = new Dag(ImmutableSet.of(new Connection("n3.connector", "n3"), new Connection("n3", "n4"), new Connection("n4", "condition")));
    Dag dag3 = new Dag(ImmutableSet.of(new Connection("condition", "n5")));
    Dag dag4 = new Dag(ImmutableSet.of(new Connection("condition", "n6")));
    Set<Dag> expected = ImmutableSet.of(dag1, dag2, dag3, dag4);
    Assert.assertEquals(actual, expected);
}