Example 26 with Connection
use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.
the class ControlDagTest method testFlatten.
@Test
public void testFlatten() {
/*
|--> n3
|--> n2 --|
| |--> n4
n1 --|
|
|--> n5
*/
ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4")));
cdag.flatten();
/*
|--> n2 --| |--> n3 --|
n1 --| |--> n2.n5 -->| |--> n3.n4
|--> n5 --| |--> n4 --|
*/
ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n3", "n3.n4"), new Connection("n4", "n3.n4")));
Assert.assertEquals(expected, cdag);
/*
|--> n3
|--> n2 --|
| |--> n4
n1 --| |
| v
|--> n5 -----> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n4", "n6"), new Connection("n5", "n6")));
cdag.flatten();
/*
|--> n2 --|
| | |--> n3 ---------|
n1 --| |--> n2.n5 --| |--> n3.n6
| | |--> n4 --> n6 --|
|--> n5 --|
*/
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n5"), new Connection("n2", "n2.n5"), new Connection("n5", "n2.n5"), new Connection("n2.n5", "n3"), new Connection("n2.n5", "n4"), new Connection("n4", "n6"), new Connection("n3", "n3.n6"), new Connection("n6", "n3.n6")));
Assert.assertEquals(expected, cdag);
/*
|--> n2 --|
| |--> n5
n1 --|--> n3 --|
|
|--> n4 --> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n5"), new Connection("n3", "n5"), new Connection("n4", "n6")));
cdag.flatten();
/*
|--> n2 ---------|
| |
n1 --|--> n3 ---------|--> n2.n3.n6 --> n5
| |
|--> n4 --> n6 --|
*/
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n4", "n6"), new Connection("n2", "n2.n3.n6"), new Connection("n3", "n2.n3.n6"), new Connection("n6", "n2.n3.n6"), new Connection("n2.n3.n6", "n5")));
Assert.assertEquals(expected, cdag);
}
Also used :
Connection(co.cask.cdap.etl.proto.Connection)
Test(org.junit.Test)
Example 27 with Connection
use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.
the class ControlDagTest method testTrim.
@Test
public void testTrim() {
/*
|--> n2 --|
n1 --| v
|-------> n3
*/
ControlDag cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n2", "n3")));
/*
trims down to:
n1 --> n2 --> n3
*/
Assert.assertEquals(1, cdag.trim());
ControlDag expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3")));
Assert.assertEquals(expected, cdag);
/*
|--> n3 --|
|--> n2 --| |--> n5
| |--> n4 --| ^
n1 --| | |
| v |
|--> n6 -----> n7 -------|
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n3", "n5"), new Connection("n4", "n5"), new Connection("n4", "n7"), new Connection("n6", "n7"), new Connection("n7", "n5")));
/*
trims down to:
|--> n3 --> n5
|--> n2 --| ^
| |--> n4 |
n1 --| | |
| v |
|--> n6 -----> n7 ----|
*/
Assert.assertEquals(1, cdag.trim());
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n2", "n4"), new Connection("n3", "n5"), new Connection("n4", "n7"), new Connection("n6", "n7"), new Connection("n7", "n5")));
Assert.assertEquals(expected, cdag);
/*
|--> n2 --> n3 ------------
n1 --| | | |
| v v |
|---------> n4 --> n5 ----|
| | v
|-----------------------> n6
*/
cdag = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n4"), new Connection("n1", "n6"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n3", "n5"), new Connection("n3", "n6"), new Connection("n4", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6")));
/*
trims down to:
n1 --> n2 --> n3 --> n4 --> n5 --> n6
*/
Assert.assertEquals(5, cdag.trim());
expected = new ControlDag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6")));
Assert.assertEquals(expected, cdag);
}
Also used :
Connection(co.cask.cdap.etl.proto.Connection)
Test(org.junit.Test)
Example 28 with Connection
use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.
the class DagTest method testSubsetAround.
@Test
public void testSubsetAround() {
/*
n1 --> n2 --|
|--> n3 --> n4 --|
n7 --> n8 --| |--> n5 --> n6
|
n9 --------------------------|
*/
Dag fullDag = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n7", "n8"), new Connection("n8", "n3"), new Connection("n9", "n5")));
// test without stop nodes
/*
n1 --> n2 --|
|--> n3 --> n4 --|
|--> n5 --> n6
*/
Dag expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6")));
Assert.assertEquals(expected, fullDag.subsetAround("n2", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
Assert.assertEquals(expected, fullDag.subsetAround("n1", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
/*
n1 --> n2 --|
|--> n3 --> n4 --|
n7 --> n8 --| |--> n5 --> n6
*/
expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n7", "n8"), new Connection("n8", "n3")));
Assert.assertEquals(expected, fullDag.subsetAround("n3", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
Assert.assertEquals(expected, fullDag.subsetAround("n4", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
Assert.assertEquals(fullDag, fullDag.subsetAround("n5", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
Assert.assertEquals(fullDag, fullDag.subsetAround("n6", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
/*
|--> n3 --> n4 --|
n7 --> n8 --| |--> n5 --> n6
*/
expected = new Dag(ImmutableSet.of(new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n7", "n8"), new Connection("n8", "n3")));
Assert.assertEquals(expected, fullDag.subsetAround("n7", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
Assert.assertEquals(expected, fullDag.subsetAround("n8", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
/*
|--> n5 --> n6
|
n9 --------------------------|
*/
expected = new Dag(ImmutableSet.of(new Connection("n5", "n6"), new Connection("n9", "n5")));
Assert.assertEquals(expected, fullDag.subsetAround("n9", ImmutableSet.<String>of(), ImmutableSet.<String>of()));
// test with stop nodes
/*
n1 --> n2 --|
|--> n3 --> n4
n8 --|
*/
expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n8", "n3")));
Assert.assertEquals(expected, fullDag.subsetAround("n3", ImmutableSet.of("n4"), ImmutableSet.of("n1", "n8")));
/*
n1 --> n2 --|
|--> n3 --> n4
*/
expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n3"), new Connection("n3", "n4")));
Assert.assertEquals(expected, fullDag.subsetAround("n2", ImmutableSet.of("n4"), ImmutableSet.of("n1", "n8")));
Assert.assertEquals(expected, fullDag.subsetAround("n1", ImmutableSet.of("n4"), ImmutableSet.of("n1", "n8")));
/*
n3 --> n4 --|
|--> n5 --> n6
|
n9 --------------------------|
*/
expected = new Dag(ImmutableSet.of(new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n9", "n5")));
Assert.assertEquals(expected, fullDag.subsetAround("n5", ImmutableSet.of("n6"), ImmutableSet.of("n3", "n9")));
Assert.assertEquals(expected, fullDag.subsetAround("n6", ImmutableSet.of("n6"), ImmutableSet.of("n3", "n9")));
/*
n2 --|
|--> n3 --> n4 --|
n8 --| |--> n5
|
n9 --------------------------|
*/
expected = new Dag(ImmutableSet.of(new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n5", "n6"), new Connection("n8", "n3"), new Connection("n9", "n5")));
Assert.assertEquals(expected, fullDag.subsetAround("n5", ImmutableSet.of("n6"), ImmutableSet.of("n2", "n8")));
/*
n2 --|
|--> n3 --> n4 --|
n8 --| |--> n5
*/
expected = new Dag(ImmutableSet.of(new Connection("n2", "n3"), new Connection("n3", "n4"), new Connection("n4", "n5"), new Connection("n8", "n3")));
Assert.assertEquals(expected, fullDag.subsetAround("n4", ImmutableSet.of("n5"), ImmutableSet.of("n2", "n8")));
}
Also used :
Connection(co.cask.cdap.etl.proto.Connection)
Test(org.junit.Test)
Example 29 with Connection
use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.
the class DagTest method testSubset.
@Test
public void testSubset() {
/*
n1 -- n2
|
v
n3 -- n4 --- n8
^
|
n5-------- n6 -- n7
*/
Dag fulldag = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n4"), new Connection("n3", "n4"), new Connection("n4", "n8"), new Connection("n5", "n4"), new Connection("n5", "n6"), new Connection("n6", "n7")));
Dag expected = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n2", "n4"), new Connection("n4", "n8")));
Dag actual = fulldag.subsetFrom("n1");
Assert.assertEquals(expected, actual);
expected = new Dag(ImmutableSet.of(new Connection("n2", "n4"), new Connection("n4", "n8")));
actual = fulldag.subsetFrom("n2");
Assert.assertEquals(expected, actual);
expected = new Dag(ImmutableSet.of(new Connection("n3", "n4"), new Connection("n4", "n8")));
actual = fulldag.subsetFrom("n3");
Assert.assertEquals(expected, actual);
expected = new Dag(ImmutableSet.of(new Connection("n4", "n8"), new Connection("n5", "n4"), new Connection("n5", "n6"), new Connection("n6", "n7")));
actual = fulldag.subsetFrom("n5");
Assert.assertEquals(expected, actual);
expected = new Dag(ImmutableSet.of(new Connection("n6", "n7")));
actual = fulldag.subsetFrom("n6");
Assert.assertEquals(expected, actual);
// test subsets with stop nodes
expected = new Dag(ImmutableSet.of(new Connection("n1", "n2")));
actual = fulldag.subsetFrom("n1", ImmutableSet.of("n2"));
Assert.assertEquals(expected, actual);
expected = new Dag(ImmutableSet.of(new Connection("n5", "n4"), new Connection("n5", "n6")));
actual = fulldag.subsetFrom("n5", ImmutableSet.of("n4", "n6"));
Assert.assertEquals(expected, actual);
/*
|--- n2 ----------|
| | |-- n10
n1 --|--- n3 --- n5 ---|--- n6 --- n7 --- n8 --- n9 --|
| | |-- n11
|--- n4 ----------|
*/
fulldag = new Dag(ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n6"), new Connection("n3", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9"), new Connection("n9", "n10"), new Connection("n9", "n11")));
expected = new Dag(ImmutableSet.of(new Connection("n3", "n5"), new Connection("n5", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9")));
actual = fulldag.subsetFrom("n3", ImmutableSet.of("n4", "n9"));
Assert.assertEquals(expected, actual);
expected = new Dag(ImmutableSet.of(new Connection("n2", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8")));
actual = fulldag.subsetFrom("n2", ImmutableSet.of("n4", "n8", "n1"));
Assert.assertEquals(expected, actual);
}
Also used :
Connection(co.cask.cdap.etl.proto.Connection)
Test(org.junit.Test)
Example 30 with Connection
use of co.cask.cdap.etl.proto.Connection in project cdap by caskdata.
the class DagTest method testIdentitySplitByControl.
@Test
public void testIdentitySplitByControl() {
// |-- n0 --|
// a0 --| |-- n2
// |-- n1 --|
Dag dag = new Dag(ImmutableSet.of(new Connection("a0", "n0"), new Connection("a0", "n1"), new Connection("n0", "n2"), new Connection("n1", "n2")));
Set<Dag> actual = dag.splitByControlNodes(ImmutableSet.<String>of(), ImmutableSet.of("a0"));
Set<Dag> expectedDags = new HashSet<>();
expectedDags.add(dag);
Assert.assertEquals(expectedDags, actual);
// a0 -- n0 --|
// |-- n2
// n1 --|
dag = new Dag(ImmutableSet.of(new Connection("a0", "n0"), new Connection("n0", "n2"), new Connection("n1", "n2")));
actual = dag.splitByControlNodes(ImmutableSet.<String>of(), ImmutableSet.of("a0"));
expectedDags.clear();
expectedDags.add(dag);
Assert.assertEquals(expectedDags, actual);
// a0 -- n0 -- a1
dag = new Dag(ImmutableSet.of(new Connection("a0", "n0"), new Connection("n0", "a1")));
actual = dag.splitByControlNodes(ImmutableSet.<String>of(), ImmutableSet.of("a0", "a1"));
expectedDags.clear();
expectedDags.add(dag);
Assert.assertEquals(expectedDags, actual);
// n0 --|
// |-- a0
// |---|
// n1
// |---|
// |-- n2
// a1 --|
dag = new Dag(ImmutableSet.of(new Connection("n0", "a0"), new Connection("n1", "a0"), new Connection("n1", "n2"), new Connection("a1", "n2")));
actual = dag.splitByControlNodes(ImmutableSet.<String>of(), ImmutableSet.of("a0", "a1"));
expectedDags.clear();
expectedDags.add(dag);
Assert.assertEquals(expectedDags, actual);
}
Also used :
Connection(co.cask.cdap.etl.proto.Connection)
HashSet(java.util.HashSet)
Test(org.junit.Test)
Aggregations
Connection (co.cask.cdap.etl.proto.Connection)36
Test (org.junit.Test)26
HashSet (java.util.HashSet)23
HashMap (java.util.HashMap)10
StageSpec (co.cask.cdap.etl.spec.StageSpec)8
PipelinePhase (co.cask.cdap.etl.common.PipelinePhase)6
PipelineSpec (co.cask.cdap.etl.spec.PipelineSpec)5
Resources (co.cask.cdap.api.Resources)4
UpgradeContext (co.cask.cdap.etl.proto.UpgradeContext)4
ArrayList (java.util.ArrayList)4
ArtifactSelectorConfig (co.cask.cdap.etl.proto.ArtifactSelectorConfig)3
Dag (co.cask.cdap.etl.planner.Dag)2
Plugin (co.cask.cdap.etl.proto.v1.Plugin)2
ETLPlugin (co.cask.cdap.etl.proto.v2.ETLPlugin)2
ImmutableSet (com.google.common.collect.ImmutableSet)2
Map (java.util.Map)2
Set (java.util.Set)2
Schema (co.cask.cdap.api.data.schema.Schema)1
PartitionedFileSet (co.cask.cdap.api.dataset.lib.PartitionedFileSet)1
ConditionBranches (co.cask.cdap.etl.planner.ConditionBranches)1
