Examples with DAGImpl org.apache.tez.dag.app.dag.impl.DAGImpl used on opensource projects

Example 6 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testBasicEvents.

@Test(timeout = 5000)
public void testBasicEvents() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestEventsDelegate();
    DAG dag = DAG.create("testBasicEvents");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vC = Vertex.create("C", ProcessorDescriptor.create("Proc.class"), 2);
    Vertex vD = Vertex.create("D", ProcessorDescriptor.create("Proc.class"), 2);
    dag.addVertex(vA).addVertex(vB).addVertex(vC).addVertex(vD).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vC, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")))).addEdge(Edge.create(vA, vD, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    VertexImpl vImpl = (VertexImpl) dagImpl.getVertex(vB.getName());
    TaskImpl tImpl = (TaskImpl) vImpl.getTask(1);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    List<TezEvent> tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    int targetIndex1 = ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    int targetIndex2 = ((DataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vC.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 2 from vA
    Assert.assertEquals(2, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex());
    targetIndex1 = ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex();
    targetIndex2 = ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex();
    // order of vA task completion can change order of events
    Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0));
    vImpl = (VertexImpl) dagImpl.getVertex(vD.getName());
    tImpl = (TaskImpl) vImpl.getTask(1);
    taId = TezTaskAttemptID.getInstance(tImpl.getTaskId(), 0);
    tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents();
    // 1 from vA
    Assert.assertEquals(1, tEvents.size());
    Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex());
    Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex());
    tezClient.stop();
}

Example 7 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounters.

@Test(timeout = 10000)
public void testBasicCounters() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String procCounterName = "Proc";
    final String globalCounterName = "Global";
    DAG dag = DAG.create("testBasicCounters");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    TezCounters temp = new TezCounters();
    temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutput out = new DataOutputStream(bos);
    temp.write(out);
    final byte[] payload = bos.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            String vName = taskSpec.getVertexName();
            TezCounters counters = new TezCounters();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                // the internal merges of counters covers the constructor code path.
                counters.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            counters.findCounter(vName, procCounterName).increment(1);
            for (OutputSpec output : taskSpec.getOutputs()) {
                counters.findCounter(vName, output.getDestinationVertexName()).increment(1);
            }
            for (InputSpec input : taskSpec.getInputs()) {
                counters.findCounter(vName, input.getSourceVertexName()).increment(1);
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
    if (SystemUtils.IS_OS_LINUX) {
        Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0);
    }
    // verify processor counters
    Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue());
    // verify edge counters
    Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue());
    // verify global counters
    Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue());
    VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName);
    VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName);
    TezCounters vACounters = vAImpl.getAllCounters();
    TezCounters vBCounters = vBImpl.getAllCounters();
    String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName();
    String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName();
    if (vACounterName != vBCounterName) {
        Assert.fail("String counter name objects dont match despite interning.");
    }
    CounterGroup vaGroup = vACounters.getGroup(globalCounterName);
    String vaGrouName = vaGroup.getName();
    CounterGroup vBGroup = vBCounters.getGroup(globalCounterName);
    String vBGrouName = vBGroup.getName();
    if (vaGrouName != vBGrouName) {
        Assert.fail("String group name objects dont match despite interning.");
    }
    tezClient.stop();
}

Example 8 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatisticsMemory.

@Ignore
@Test(timeout = 60000)
public void testBasicStatisticsMemory() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "abcdefghijklmnopqrstuvwxyz";
    int numTasks = 10000;
    int numSources = 10;
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    DAG dag = DAG.create("testBasicStatisticsMemory");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), numTasks);
    for (int i = 0; i < numSources; ++i) {
        final String sourceName = i + vAName;
        vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create(sourceName), null, null));
        vAStats.addIO(sourceName, ioStats);
    }
    dag.addVertex(vA);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getDataSize());
    Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getItemsProcessed());
    checkMemory(dag.getName(), mockApp);
    tezClient.stop();
}

Example 9 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestPreemption method testPreemptionJob.

void testPreemptionJob(MockTezClient tezClient, DAG dag, int vertexIndex, int upToTaskVersion, String info) throws Exception {
    System.out.println("TestPreemption - Running - " + info);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0);
    // turn off scheduling to block DAG before submitting it
    mockLauncher.startScheduling(false);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    for (int i = 0; i <= upToTaskVersion; ++i) {
        TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i);
        TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId);
        Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
        Assert.assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, taImpl.getTerminationCause());
    }
    System.out.println("TestPreemption - Done running - " + info);
}

Example 10 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestPreemption method testPreemptionWithoutSession.

@Test(timeout = 5000)
public void testPreemptionWithoutSession() throws Exception {
    System.out.println("TestPreemptionWithoutSession");
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0);
    AtomicBoolean mockAppLauncherGoFlag = new AtomicBoolean(false);
    MockTezClient tezClient = new MockTezClient("testPreemption", tezconf, false, null, null, null, mockAppLauncherGoFlag, false, false, 2, 2);
    tezClient.start();
    DAGClient dagClient = tezClient.submitDAG(createDAG(DataMovementType.SCATTER_GATHER));
    // now the MockApp has been started. sync with it to get the launcher
    syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient);
    DAGImpl dagImpl;
    do {
        // usually needs to sleep 2-3 times
        Thread.sleep(100);
    } while ((dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG()) == null);
    int vertexIndex = 0;
    int upToTaskVersion = 3;
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    for (int i = 0; i <= upToTaskVersion; ++i) {
        TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i);
        TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId);
        Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
    }
    tezClient.stop();
}