Examples with DAGStatus org.apache.tez.dag.api.client.DAGStatus used on opensource projects

Example 26 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMemoryWithEvents method testMemory.

private void testMemory(DAG dag, boolean sendDMEvents) throws Exception {
    StopWatch stopwatch = new StopWatch();
    stopwatch.start();
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, numThreads, 1000);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.eventsDelegate = new TestMockDAGAppMaster.TestEventsDelegate();
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    checkMemory(dag.getName(), mockApp);
    stopwatch.stop();
    System.out.println("Time taken(ms): " + stopwatch.now(TimeUnit.MILLISECONDS));
    tezClient.stop();
}

Example 27 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatistics.

@Test(timeout = 10000)
public void testBasicStatistics() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String sourceName = "In";
    final String sinkName = "Out";
    DAG dag = DAG.create("testBasisStatistics");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 3);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 2);
    vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create("In"), null, null));
    vB.addDataSink(sinkName, DataSinkDescriptor.create(OutputDescriptor.create("Out"), null, null));
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    vAStats.addIO(vBName, ioStats);
    vAStats.addIO(sourceName, ioStats);
    TaskStatistics vBStats = new TaskStatistics();
    vBStats.addIO(vAName, ioStats);
    vBStats.addIO(sinkName, ioStats);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    ByteArrayOutputStream bosB = new ByteArrayOutputStream();
    DataOutput outB = new DataOutputStream(bosB);
    vBStats.write(outB);
    final byte[] payloadB = bosB.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            if (taskSpec.getVertexName().equals(vBName)) {
                payload = payloadB;
            }
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    // verify that the values have been correct aggregated
    for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) {
        VertexStatistics vStats = v.getStatistics();
        if (v.getName().equals(vAName)) {
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed());
        } else {
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed());
        }
    }
    tezClient.stop();
}

Example 28 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMockDAGAppMaster method testTaskEventsProcessingSpeed.

@Ignore
@Test(timeout = 60000)
public void testTaskEventsProcessingSpeed() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setBoolean(TezConfiguration.TEZ_AM_USE_CONCURRENT_DISPATCHER, true);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, 30, 1000);
    tezClient.start();
    final String vAName = "A";
    DAG dag = DAG.create("testTaskEventsProcessingSpeed");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 50000);
    dag.addVertex(vA);
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    tezClient.stop();
}

Example 29 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounterMemory.

@Ignore
@Test(timeout = 60000)
public void testBasicCounterMemory() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    DAG dag = DAG.create("testBasicCounterMemory");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10000);
    dag.addVertex(vA);
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            TezCounters counters = new TezCounters();
            final String longName = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
            final String shortName = "abcdefghijklmnopqrstuvwxyz";
            for (int i = 0; i < 6; ++i) {
                for (int j = 0; j < 15; ++j) {
                    counters.findCounter((i + longName), (i + (shortName))).increment(1);
                }
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    Assert.assertNotNull(counters);
    checkMemory(dag.getName(), mockApp);
    tezClient.stop();
}

Example 30 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class YARNRunner method getJobStatus.

@Override
public JobStatus getJobStatus(JobID jobID) throws IOException, InterruptedException {
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    String jobFile = MRApps.getJobFile(conf, user, jobID);
    DAGStatus dagStatus;
    try {
        if (dagClient == null) {
            dagClient = MRTezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId(), tezConf, null);
        }
        dagStatus = dagClient.getDAGStatus(null);
        return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
    } catch (TezException e) {
        throw new IOException(e);
    }
}