use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.
the class TestMockDAGAppMaster method testMixedEdgeRouting.
@Test(timeout = 100000)
public void testMixedEdgeRouting() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
tezClient.start();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
mockApp.eventsDelegate = new TestEventsDelegate();
DAG dag = DAG.create("testMixedEdgeRouting");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 1);
Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 1);
Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 1);
Vertex vE = Vertex.create("E", ProcessorDescriptor.create("Proc.class"), 1);
dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addVertex(vE).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vD, EdgeProperty.create(EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vE, EdgeProperty.create(EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
DAGClient dagClient = tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
// vC uses on demand routing and its task does not provide events
VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
TaskImpl tImpl = (TaskImpl) vImpl.getTask(0);
TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
Assert.assertEquals(0, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
// vD is mixed mode and only 1 out of 2 edges does legacy routing with task providing events
vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
tImpl = (TaskImpl) vImpl.getTask(0);
taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
Assert.assertEquals(1, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
// vE has single legacy edge and does not use on demand routing and its task provides events
vImpl = (VertexImpl) dagImpl.getVertex(vE.getName());
tImpl = (TaskImpl) vImpl.getTask(0);
taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
Assert.assertEquals(1, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
tezClient.stop();
}
use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.
the class TestMockDAGAppMaster method testInternalPreemption.
@Test(timeout = 5000)
public void testInternalPreemption() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
tezClient.start();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
// there is only 1 task whose first attempt will be preempted
DAG dag = DAG.create("testInternalPreemption");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
dag.addVertex(vA);
DAGClient dagClient = tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
ContainerData cData = mockLauncher.getContainers().values().iterator().next();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
TaskAttempt killedTa = dagImpl.getVertex(vA.getName()).getTask(0).getAttempt(killedTaId);
Assert.assertEquals(TaskAttemptState.KILLED, killedTa.getState());
tezClient.stop();
}
use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.
the class TestMockDAGAppMaster method testBasicEvents.
@Test(timeout = 5000)
public void testBasicEvents() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
tezClient.start();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
mockApp.eventsDelegate = new TestEventsDelegate();
DAG dag = DAG.create("testBasicEvents");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 2);
Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 2);
Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 2);
Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 2);
dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
DAGClient dagClient = tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vB.getName());
TaskImpl tImpl = (TaskImpl) vImpl.getTask(1);
TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
List<TezEvent> tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
// 2 from vA
Assert.assertEquals(2, tEvents.size());
Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
int targetIndex1 = ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
int targetIndex2 = ((DataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
// order of vA task completion can change order of events
Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
tImpl = (TaskImpl) vImpl.getTask(1);
taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
// 2 from vA
Assert.assertEquals(2, tEvents.size());
Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
targetIndex1 = ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
targetIndex2 = ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
// order of vA task completion can change order of events
Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
tImpl = (TaskImpl) vImpl.getTask(1);
taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
// 1 from vA
Assert.assertEquals(1, tEvents.size());
Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex());
Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
tezClient.stop();
}
use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.
the class TestMockDAGAppMaster method testBasicCounters.
@Test(timeout = 10000)
public void testBasicCounters() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
tezClient.start();
final String vAName = "A";
final String vBName = "B";
final String procCounterName = "Proc";
final String globalCounterName = "Global";
DAG dag = DAG.create("testBasicCounters");
Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10);
Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1);
dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
TezCounters temp = new TezCounters();
temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutput out = new DataOutputStream(bos);
temp.write(out);
final byte[] payload = bos.toByteArray();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
mockApp.countersDelegate = new CountersDelegate() {
@Override
public TezCounters getCounters(TaskSpec taskSpec) {
String vName = taskSpec.getVertexName();
TezCounters counters = new TezCounters();
final DataInputByteBuffer in = new DataInputByteBuffer();
in.reset(ByteBuffer.wrap(payload));
try {
// this ensures that the serde code path is covered.
// the internal merges of counters covers the constructor code path.
counters.readFields(in);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
counters.findCounter(vName, procCounterName).increment(1);
for (OutputSpec output : taskSpec.getOutputs()) {
counters.findCounter(vName, output.getDestinationVertexName()).increment(1);
}
for (InputSpec input : taskSpec.getInputs()) {
counters.findCounter(vName, input.getSourceVertexName()).increment(1);
}
return counters;
}
};
mockApp.doSleep = false;
DAGClient dagClient = tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
mockLauncher.startScheduling(true);
DAGStatus status = dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
TezCounters counters = dagImpl.getAllCounters();
String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
if (SystemUtils.IS_OS_LINUX) {
Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0);
}
// verify processor counters
Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue());
Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue());
// verify edge counters
Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue());
Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue());
// verify global counters
Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue());
VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName);
VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName);
TezCounters vACounters = vAImpl.getAllCounters();
TezCounters vBCounters = vBImpl.getAllCounters();
String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName();
String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName();
if (vACounterName != vBCounterName) {
Assert.fail("String counter name objects dont match despite interning.");
}
CounterGroup vaGroup = vACounters.getGroup(globalCounterName);
String vaGrouName = vaGroup.getName();
CounterGroup vBGroup = vBCounters.getGroup(globalCounterName);
String vBGrouName = vBGroup.getName();
if (vaGrouName != vBGrouName) {
Assert.fail("String group name objects dont match despite interning.");
}
tezClient.stop();
}
use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.
the class TestMockDAGAppMaster method testSchedulerErrorHandling.
@Test(timeout = 10000)
public void testSchedulerErrorHandling() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
tezClient.start();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
DAG dag = DAG.create("testSchedulerErrorHandling");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
dag.addVertex(vA);
tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
mockApp.handle(new DAGAppMasterEventSchedulingServiceError(org.apache.hadoop.util.StringUtils.stringifyException(new RuntimeException("Mock error"))));
while (!mockApp.getShutdownHandler().wasShutdownInvoked()) {
Thread.sleep(100);
}
Assert.assertEquals(DAGState.RUNNING, mockApp.getContext().getCurrentDAG().getState());
}
Aggregations