Examples with DAGImpl org.apache.tez.dag.app.dag.impl.DAGImpl used on opensource projects

Example 11 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestSpeculation method testBasicSpeculationPerVertexConf.

@Test(timeout = 10000)
public void testBasicSpeculationPerVertexConf() throws Exception {
    DAG dag = DAG.create("test");
    String vNameNoSpec = "A";
    String vNameSpec = "B";
    Vertex vA = Vertex.create(vNameNoSpec, ProcessorDescriptor.create("Proc.class"), 5);
    Vertex vB = Vertex.create(vNameSpec, ProcessorDescriptor.create("Proc.class"), 5);
    vA.setConf(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, "false");
    dag.addVertex(vA);
    dag.addVertex(vB);
    // min/max src fraction is set to 1. So vertices will run sequentially
    dag.addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("O"), InputDescriptor.create("I"))));
    MockTezClient tezClient = createTezSession();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = dagImpl.getVertex(vNameSpec).getVertexId();
    TezVertexID vertexIdNoSpec = dagImpl.getVertex(vNameNoSpec).getVertexId();
    // original attempt is killed and speculative one is successful
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID noSpecTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexIdNoSpec, 0), 0);
    // cause speculation trigger for both
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
    mockLauncher.setStatusUpdatesForTask(noSpecTaId, 100);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    org.apache.tez.dag.app.dag.Vertex vSpec = dagImpl.getVertex(vertexId);
    org.apache.tez.dag.app.dag.Vertex vNoSpec = dagImpl.getVertex(vertexIdNoSpec);
    // speculation for vA but not for vB
    Assert.assertTrue(vSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue() > 0);
    Assert.assertEquals(0, vNoSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    tezClient.stop();
}

Example 12 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestDAGAppMaster method testDagCredentials.

@SuppressWarnings("deprecation")
private void testDagCredentials(boolean doMerge) throws IOException {
    TezConfiguration conf = new TezConfiguration();
    conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge);
    conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString());
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    // create some sample AM credentials
    Credentials amCreds = new Credentials();
    JobTokenSecretManager jtsm = new JobTokenSecretManager();
    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString()));
    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm);
    sessionToken.setService(identifier.getJobId());
    TokenCache.setSessionToken(sessionToken, amCreds);
    TestTokenSecretManager ttsm = new TestTokenSecretManager();
    Text tokenAlias1 = new Text("alias1");
    Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("amtoken1")), ttsm);
    amCreds.addToken(tokenAlias1, amToken1);
    Text tokenAlias2 = new Text("alias2");
    Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("amtoken2")), ttsm);
    amCreds.addToken(tokenAlias2, amToken2);
    FileSystem fs = FileSystem.getLocal(conf);
    FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
    DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream);
    sessionJarsPBOutStream.close();
    DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), amCreds, "someuser", null);
    am.init(conf);
    am.start();
    // create some sample DAG credentials
    Credentials dagCreds = new Credentials();
    Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("dagtoken1")), ttsm);
    dagCreds.addToken(tokenAlias2, dagToken1);
    Text tokenAlias3 = new Text("alias3");
    Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>(new TestTokenIdentifier(new Text("dagtoken2")), ttsm);
    dagCreds.addToken(tokenAlias3, dagToken2);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag").setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build();
    DAGImpl dag = am.createDAG(dagPlan, dagId);
    Credentials fetchedDagCreds = dag.getCredentials();
    am.stop();
    Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1);
    if (doMerge) {
        assertNotNull("AM creds missing from DAG creds", fetchedToken1);
        compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1));
    } else {
        assertNull("AM creds leaked to DAG creds", fetchedToken1);
    }
    compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2));
    compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3));
}

Example 13 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatistics.

@Test(timeout = 10000)
public void testBasicStatistics() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String sourceName = "In";
    final String sinkName = "Out";
    DAG dag = DAG.create("testBasisStatistics");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 3);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 2);
    vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create("In"), null, null));
    vB.addDataSink(sinkName, DataSinkDescriptor.create(OutputDescriptor.create("Out"), null, null));
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    vAStats.addIO(vBName, ioStats);
    vAStats.addIO(sourceName, ioStats);
    TaskStatistics vBStats = new TaskStatistics();
    vBStats.addIO(vAName, ioStats);
    vBStats.addIO(sinkName, ioStats);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    ByteArrayOutputStream bosB = new ByteArrayOutputStream();
    DataOutput outB = new DataOutputStream(bosB);
    vBStats.write(outB);
    final byte[] payloadB = bosB.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            if (taskSpec.getVertexName().equals(vBName)) {
                payload = payloadB;
            }
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    // verify that the values have been correct aggregated
    for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) {
        VertexStatistics vStats = v.getStatistics();
        if (v.getName().equals(vAName)) {
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed());
        } else {
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed());
        }
    }
    tezClient.stop();
}

Example 14 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounterMemory.

@Ignore
@Test(timeout = 60000)
public void testBasicCounterMemory() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    DAG dag = DAG.create("testBasicCounterMemory");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10000);
    dag.addVertex(vA);
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            TezCounters counters = new TezCounters();
            final String longName = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
            final String shortName = "abcdefghijklmnopqrstuvwxyz";
            for (int i = 0; i < 6; ++i) {
                for (int j = 0; j < 15; ++j) {
                    counters.findCounter((i + longName), (i + (shortName))).increment(1);
                }
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    Assert.assertNotNull(counters);
    checkMemory(dag.getName(), mockApp);
    tezClient.stop();
}

Example 15 with DAGImpl

use of org.apache.tez.dag.app.dag.impl.DAGImpl in project tez by apache.

the class DAGAppMaster method createDAG.

/**
 * Create and initialize (but don't start) a single dag.
 */
DAGImpl createDAG(DAGPlan dagPB, TezDAGID dagId) {
    if (dagId == null) {
        dagId = TezDAGID.getInstance(appAttemptID.getApplicationId(), dagCounter.incrementAndGet());
    }
    Credentials dagCredentials = null;
    if (dagPB.hasCredentialsBinary()) {
        dagCredentials = DagTypeConverters.convertByteStringToCredentials(dagPB.getCredentialsBinary());
        TezCommonUtils.logCredentials(LOG, dagCredentials, "dag");
    } else {
        dagCredentials = new Credentials();
    }
    if (getConfig().getBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, TezConfiguration.TEZ_AM_CREDENTIALS_MERGE_DEFAULT)) {
        LOG.info("Merging AM credentials into DAG credentials");
        dagCredentials.mergeAll(amCredentials);
    }
    // TODO Does this move to the client in case of work-preserving recovery.
    TokenCache.setSessionToken(sessionToken, dagCredentials);
    // create single dag
    DAGImpl newDag = new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(), taskCommunicatorManager, dagCredentials, clock, appMasterUgi.getShortUserName(), taskHeartbeatHandler, context);
    try {
        if (LOG.isDebugEnabled()) {
            LOG.debug("JSON dump for submitted DAG, dagId=" + dagId.toString() + ", json=" + DAGUtils.generateSimpleJSONPlan(dagPB).toString());
        }
    } catch (JSONException e) {
        LOG.warn("Failed to generate json for DAG", e);
    }
    generateDAGVizFile(dagId, dagPB, logDirs);
    writePBTextFile(newDag);
    return newDag;
}