Search in sources :

Example 1 with MockContainerLauncher

use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.

the class TestMockDAGAppMaster method testMixedEdgeRouting.

@Test(timeout = 100000)
public void testMixedEdgeRouting() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestEventsDelegate();
    DAG dag = DAG.create("testMixedEdgeRouting");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vE = Vertex.create("E", ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addVertex(vE).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vD, EdgeProperty.create(EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vE, EdgeProperty.create(EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    // vC uses on demand routing and its task does not provide events
    VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
    TaskImpl tImpl = (TaskImpl) vImpl.getTask(0);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    Assert.assertEquals(0, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
    // vD is mixed mode and only 1 out of 2 edges does legacy routing with task providing events
    vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
    tImpl = (TaskImpl) vImpl.getTask(0);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    Assert.assertEquals(1, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
    // vE has single legacy edge and does not use on demand routing and its task provides events
    vImpl = (VertexImpl) dagImpl.getVertex(vE.getName());
    tImpl = (TaskImpl) vImpl.getTask(0);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    Assert.assertEquals(1, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
    tezClient.stop();
}
Also used : Vertex(org.apache.tez.dag.api.Vertex) TaskImpl(org.apache.tez.dag.app.dag.impl.TaskImpl) DAG(org.apache.tez.dag.api.DAG) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) DAGImpl(org.apache.tez.dag.app.dag.impl.DAGImpl) DAGClient(org.apache.tez.dag.api.client.DAGClient) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) VertexImpl(org.apache.tez.dag.app.dag.impl.VertexImpl) TezTaskAttemptID(org.apache.tez.dag.records.TezTaskAttemptID) Test(org.junit.Test)

Example 2 with MockContainerLauncher

use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.

the class TestMockDAGAppMaster method testInternalPreemption.

@Test(timeout = 5000)
public void testInternalPreemption() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    // there is only 1 task whose first attempt will be preempted
    DAG dag = DAG.create("testInternalPreemption");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA);
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    ContainerData cData = mockLauncher.getContainers().values().iterator().next();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TaskAttempt killedTa = dagImpl.getVertex(vA.getName()).getTask(0).getAttempt(killedTaId);
    Assert.assertEquals(TaskAttemptState.KILLED, killedTa.getState());
    tezClient.stop();
}
Also used : Vertex(org.apache.tez.dag.api.Vertex) DAGImpl(org.apache.tez.dag.app.dag.impl.DAGImpl) DAGClient(org.apache.tez.dag.api.client.DAGClient) DAG(org.apache.tez.dag.api.DAG) TaskAttempt(org.apache.tez.dag.app.dag.TaskAttempt) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) TezVertexID(org.apache.tez.dag.records.TezVertexID) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) ContainerData(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher.ContainerData) TezTaskAttemptID(org.apache.tez.dag.records.TezTaskAttemptID) Test(org.junit.Test)

Example 3 with MockContainerLauncher

use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.

the class TestMockDAGAppMaster method testBasicEvents.

@Test(timeout = 5000)
public void testBasicEvents() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestEventsDelegate();
    DAG dag = DAG.create("testBasicEvents");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 2);
    dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vB.getName());
    TaskImpl tImpl = (TaskImpl) vImpl.getTask(1);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    List<TezEvent> tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    int targetIndex1 = ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    int targetIndex2 = ((DataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    targetIndex1 = ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    targetIndex2 = ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 1 from vA
    Assert.assertEquals(1, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    tezClient.stop();
}
Also used : Vertex(org.apache.tez.dag.api.Vertex) TaskImpl(org.apache.tez.dag.app.dag.impl.TaskImpl) DAG(org.apache.tez.dag.api.DAG) DataMovementEvent(org.apache.tez.runtime.api.events.DataMovementEvent) CompositeRoutedDataMovementEvent(org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent) CompositeDataMovementEvent(org.apache.tez.runtime.api.events.CompositeDataMovementEvent) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) DAGImpl(org.apache.tez.dag.app.dag.impl.DAGImpl) DAGClient(org.apache.tez.dag.api.client.DAGClient) TezEvent(org.apache.tez.runtime.api.impl.TezEvent) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) VertexImpl(org.apache.tez.dag.app.dag.impl.VertexImpl) TezTaskAttemptID(org.apache.tez.dag.records.TezTaskAttemptID) Test(org.junit.Test)

Example 4 with MockContainerLauncher

use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounters.

@Test(timeout = 10000)
public void testBasicCounters() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String procCounterName = "Proc";
    final String globalCounterName = "Global";
    DAG dag = DAG.create("testBasicCounters");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    TezCounters temp = new TezCounters();
    temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutput out = new DataOutputStream(bos);
    temp.write(out);
    final byte[] payload = bos.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            String vName = taskSpec.getVertexName();
            TezCounters counters = new TezCounters();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                // the internal merges of counters covers the constructor code path.
                counters.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            counters.findCounter(vName, procCounterName).increment(1);
            for (OutputSpec output : taskSpec.getOutputs()) {
                counters.findCounter(vName, output.getDestinationVertexName()).increment(1);
            }
            for (InputSpec input : taskSpec.getInputs()) {
                counters.findCounter(vName, input.getSourceVertexName()).increment(1);
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
    if (SystemUtils.IS_OS_LINUX) {
        Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0);
    }
    // verify processor counters
    Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue());
    // verify edge counters
    Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue());
    // verify global counters
    Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue());
    VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName);
    VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName);
    TezCounters vACounters = vAImpl.getAllCounters();
    TezCounters vBCounters = vBImpl.getAllCounters();
    String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName();
    String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName();
    if (vACounterName != vBCounterName) {
        Assert.fail("String counter name objects dont match despite interning.");
    }
    CounterGroup vaGroup = vACounters.getGroup(globalCounterName);
    String vaGrouName = vaGroup.getName();
    CounterGroup vBGroup = vBCounters.getGroup(globalCounterName);
    String vBGrouName = vBGroup.getName();
    if (vaGrouName != vBGrouName) {
        Assert.fail("String group name objects dont match despite interning.");
    }
    tezClient.stop();
}
Also used : Vertex(org.apache.tez.dag.api.Vertex) DataOutput(java.io.DataOutput) DataOutputStream(java.io.DataOutputStream) DataInputByteBuffer(org.apache.hadoop.io.DataInputByteBuffer) TaskSpec(org.apache.tez.runtime.api.impl.TaskSpec) CounterGroup(org.apache.tez.common.counters.CounterGroup) CountersDelegate(org.apache.tez.dag.app.MockDAGAppMaster.CountersDelegate) InputSpec(org.apache.tez.runtime.api.impl.InputSpec) DAG(org.apache.tez.dag.api.DAG) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IOException(java.io.IOException) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) TezCounters(org.apache.tez.common.counters.TezCounters) DAGImpl(org.apache.tez.dag.app.dag.impl.DAGImpl) DAGClient(org.apache.tez.dag.api.client.DAGClient) DAGStatus(org.apache.tez.dag.api.client.DAGStatus) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) OutputSpec(org.apache.tez.runtime.api.impl.OutputSpec) VertexImpl(org.apache.tez.dag.app.dag.impl.VertexImpl) Test(org.junit.Test)

Example 5 with MockContainerLauncher

use of org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher in project tez by apache.

the class TestMockDAGAppMaster method testSchedulerErrorHandling.

@Test(timeout = 10000)
public void testSchedulerErrorHandling() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    DAG dag = DAG.create("testSchedulerErrorHandling");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
    dag.addVertex(vA);
    tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    mockApp.handle(new DAGAppMasterEventSchedulingServiceError(org.apache.hadoop.util.StringUtils.stringifyException(new RuntimeException("Mock error"))));
    while (!mockApp.getShutdownHandler().wasShutdownInvoked()) {
        Thread.sleep(100);
    }
    Assert.assertEquals(DAGState.RUNNING, mockApp.getContext().getCurrentDAG().getState());
}
Also used : Vertex(org.apache.tez.dag.api.Vertex) DAG(org.apache.tez.dag.api.DAG) DAGAppMasterEventSchedulingServiceError(org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) Test(org.junit.Test)

Aggregations

TezConfiguration (org.apache.tez.dag.api.TezConfiguration)12 MockContainerLauncher (org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher)12 DAG (org.apache.tez.dag.api.DAG)11 Vertex (org.apache.tez.dag.api.Vertex)11 DAGClient (org.apache.tez.dag.api.client.DAGClient)11 Test (org.junit.Test)11 DAGImpl (org.apache.tez.dag.app.dag.impl.DAGImpl)7 DAGStatus (org.apache.tez.dag.api.client.DAGStatus)5 TaskSpec (org.apache.tez.runtime.api.impl.TaskSpec)4 ByteArrayOutputStream (java.io.ByteArrayOutputStream)3 DataOutput (java.io.DataOutput)3 DataOutputStream (java.io.DataOutputStream)3 IOException (java.io.IOException)3 DataInputByteBuffer (org.apache.hadoop.io.DataInputByteBuffer)3 VertexImpl (org.apache.tez.dag.app.dag.impl.VertexImpl)3 TezTaskAttemptID (org.apache.tez.dag.records.TezTaskAttemptID)3 TezCounters (org.apache.tez.common.counters.TezCounters)2 CountersDelegate (org.apache.tez.dag.app.MockDAGAppMaster.CountersDelegate)2 ContainerData (org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher.ContainerData)2 StatisticsDelegate (org.apache.tez.dag.app.MockDAGAppMaster.StatisticsDelegate)2