Examples with DAGImpl org.apache.tez.dag.app.dag.impl.DAGImpl used on opensource projects

Example 1 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestSpeculation method testSingleTaskSpeculation.

@Test(timeout = 10000)
public void testSingleTaskSpeculation() throws Exception {
    // Map<Timeout conf value, expected number of tasks>
    Map<Long, Integer> confToExpected = new HashMap<Long, Integer>();
    // Really long time to speculate
    confToExpected.put(Long.MAX_VALUE >> 1, 1);
    confToExpected.put(100L, 2);
    // Don't speculate
    confToExpected.put(-1L, 1);
    for (Map.Entry<Long, Integer> entry : confToExpected.entrySet()) {
        defaultConf.setLong(TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT, entry.getKey());
        DAG dag = DAG.create("test");
        Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
        dag.addVertex(vA);
        MockTezClient tezClient = createTezSession();
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
        TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
        // original attempt is killed and speculative one is successful
        TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
        TezTaskAttemptID successTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);
        Thread.sleep(200);
        // cause speculation trigger
        mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
        mockLauncher.startScheduling(true);
        dagClient.waitForCompletion();
        Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
        Task task = dagImpl.getTask(killedTaId.getTaskID());
        Assert.assertEquals(entry.getValue().intValue(), task.getAttempts().size());
        if (entry.getValue() > 1) {
            Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
            TaskAttempt killedAttempt = task.getAttempt(killedTaId);
            Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
            Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, killedAttempt.getTerminationCause());
        }
        tezClient.stop();
    }
}

Example 2 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestSpeculation method testBasicSpeculation.

public void testBasicSpeculation(boolean withProgress) throws Exception {
    DAG dag = DAG.create("test");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
    dag.addVertex(vA);
    MockTezClient tezClient = createTezSession();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    // original attempt is killed and speculative one is successful
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID successTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);
    mockLauncher.updateProgress(withProgress);
    // cause speculation trigger
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    Task task = dagImpl.getTask(killedTaId.getTaskID());
    Assert.assertEquals(2, task.getAttempts().size());
    Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
    TaskAttempt killedAttempt = task.getAttempt(killedTaId);
    Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
    Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, killedAttempt.getTerminationCause());
    if (withProgress) {
        // without progress updates occasionally more than 1 task speculates
        Assert.assertEquals(1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
        Assert.assertEquals(1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
        org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
        Assert.assertEquals(1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    }
    tezClient.stop();
}

Example 3 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestSpeculation method testBasicSpeculationNotUseful.

@Test(timeout = 10000)
public void testBasicSpeculationNotUseful() throws Exception {
    DAG dag = DAG.create("test");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
    dag.addVertex(vA);
    MockTezClient tezClient = createTezSession();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    // original attempt is successful and speculative one is killed
    TezTaskAttemptID successTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);
    mockLauncher.setStatusUpdatesForTask(successTaId, 100);
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    Task task = dagImpl.getTask(killedTaId.getTaskID());
    Assert.assertEquals(2, task.getAttempts().size());
    Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
    TaskAttempt killedAttempt = task.getAttempt(killedTaId);
    Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed speculative attempt as");
    Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_INEFFECTIVE_SPECULATION, killedAttempt.getTerminationCause());
    Assert.assertEquals(1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    Assert.assertEquals(1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
    Assert.assertEquals(1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    tezClient.stop();
}

Example 4 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testMixedEdgeRouting.

@Test(timeout = 100000)
public void testMixedEdgeRouting() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestEventsDelegate();
    DAG dag = DAG.create("testMixedEdgeRouting");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 1);
    Vertex vE = Vertex.create("E", ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addVertex(vE).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vD, EdgeProperty.create(EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vB, vE, EdgeProperty.create(EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    // vC uses on demand routing and its task does not provide events
    VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
    TaskImpl tImpl = (TaskImpl) vImpl.getTask(0);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    Assert.assertEquals(0, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
    // vD is mixed mode and only 1 out of 2 edges does legacy routing with task providing events
    vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
    tImpl = (TaskImpl) vImpl.getTask(0);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    Assert.assertEquals(1, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
    // vE has single legacy edge and does not use on demand routing and its task provides events
    vImpl = (VertexImpl) dagImpl.getVertex(vE.getName());
    tImpl = (TaskImpl) vImpl.getTask(0);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    Assert.assertEquals(1, tImpl.getTaskAttemptTezEvents(taId, 0, 1000).size());
    tezClient.stop();
}

Example 5 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testInternalPreemption.

@Test(timeout = 5000)
public void testInternalPreemption() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    // there is only 1 task whose first attempt will be preempted
    DAG dag = DAG.create("testInternalPreemption");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA);
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    ContainerData cData = mockLauncher.getContainers().values().iterator().next();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TaskAttempt killedTa = dagImpl.getVertex(vA.getName()).getTask(0).getAttempt(killedTaId);
    Assert.assertEquals(TaskAttemptState.KILLED, killedTa.getState());
    tezClient.stop();
}