Examples with DAGStatus org.apache.tez.dag.api.client.DAGStatus used on opensource projects

Example 6 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestDAGClient method testDAGStatus.

@Test(timeout = 5000)
public void testDAGStatus() throws Exception {
    DAGStatus resultDagStatus = dagClient.getDAGStatus(null);
    verify(mockProxy, times(1)).getDAGStatus(null, GetDAGStatusRequestProto.newBuilder().setDagId(dagIdStr).setTimeout(0).build());
    assertEquals(new DAGStatus(dagStatusProtoWithoutCounters, DagStatusSource.AM), resultDagStatus);
    System.out.println("DAGStatusWithoutCounter:" + resultDagStatus);
    resultDagStatus = dagClient.getDAGStatus(Sets.newSet(StatusGetOpts.GET_COUNTERS));
    verify(mockProxy, times(1)).getDAGStatus(null, GetDAGStatusRequestProto.newBuilder().setDagId(dagIdStr).setTimeout(0).addStatusOptions(StatusGetOptsProto.GET_COUNTERS).build());
    assertEquals(new DAGStatus(dagStatusProtoWithCounters, DagStatusSource.AM), resultDagStatus);
    System.out.println("DAGStatusWithCounter:" + resultDagStatus);
}

Example 7 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestATSHistoryV15 method testATSLogLevelNone.

@Test
public void testATSLogLevelNone() throws Exception {
    TezClient tezSession = null;
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "TEZ_DAG_ID");
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSV15HistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        applicationId = tezSession.getAppMasterApplicationId();
        dag.setHistoryLogLevel(HistoryLogLevel.NONE);
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
        // Verify HDFS data
        int count = verifyATSDataOnHDFS(atsActivePath, applicationId);
        Assert.assertEquals("Count is: " + count, 1, count);
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 8 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestATSHistoryV15 method testSimpleDAG.

@Test(timeout = 50000)
public void testSimpleDAG() throws Exception {
    TezClient tezSession = null;
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "TEZ_DAG_ID");
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSV15HistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        applicationId = tezSession.getAppMasterApplicationId();
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
        // Verify HDFS data
        int count = verifyATSDataOnHDFS(atsActivePath, applicationId);
        Assert.assertEquals("Count is: " + count, 2, count);
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 9 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestATSHistoryWithACLs method testDisableSessionLogging.

/**
 * Test Disable Logging for all dags in a session
 * due to failure to create domain in session start
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDisableSessionLogging() throws Exception {
    TezClient tezSession = null;
    String viewAcls = "nobody nobody_group";
    SleepProcessorConfig spConf = new SleepProcessorConfig(1);
    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
    dag.addVertex(vertex);
    DAGAccessControls accessControls = new DAGAccessControls();
    accessControls.setUsersWithViewACLs(Collections.singleton("nobody2"));
    accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group2"));
    dag.setAccessControls(accessControls);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
    tezSession.start();
    // ////submit first dag //////
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        dagStatus = dagClient.getDAGStatus(null);
    }
    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    // ////submit second dag//////
    DAG dag2 = DAG.create("TezSleepProcessor2");
    vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
    dag2.addVertex(vertex);
    accessControls = new DAGAccessControls();
    accessControls.setUsersWithViewACLs(Collections.singleton("nobody3"));
    accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group3"));
    dag2.setAccessControls(accessControls);
    dagClient = tezSession.submitDAG(dag2);
    dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        dagStatus = dagClient.getDAGStatus(null);
    }
    tezSession.stop();
}

Example 10 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounters.

@Test(timeout = 10000)
public void testBasicCounters() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String procCounterName = "Proc";
    final String globalCounterName = "Global";
    DAG dag = DAG.create("testBasicCounters");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    TezCounters temp = new TezCounters();
    temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutput out = new DataOutputStream(bos);
    temp.write(out);
    final byte[] payload = bos.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            String vName = taskSpec.getVertexName();
            TezCounters counters = new TezCounters();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                // the internal merges of counters covers the constructor code path.
                counters.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            counters.findCounter(vName, procCounterName).increment(1);
            for (OutputSpec output : taskSpec.getOutputs()) {
                counters.findCounter(vName, output.getDestinationVertexName()).increment(1);
            }
            for (InputSpec input : taskSpec.getInputs()) {
                counters.findCounter(vName, input.getSourceVertexName()).increment(1);
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
    if (SystemUtils.IS_OS_LINUX) {
        Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0);
    }
    // verify processor counters
    Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue());
    // verify edge counters
    Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue());
    // verify global counters
    Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue());
    VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName);
    VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName);
    TezCounters vACounters = vAImpl.getAllCounters();
    TezCounters vBCounters = vBImpl.getAllCounters();
    String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName();
    String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName();
    if (vACounterName != vBCounterName) {
        Assert.fail("String counter name objects dont match despite interning.");
    }
    CounterGroup vaGroup = vACounters.getGroup(globalCounterName);
    String vaGrouName = vaGroup.getName();
    CounterGroup vBGroup = vBCounters.getGroup(globalCounterName);
    String vBGrouName = vBGroup.getName();
    if (vaGrouName != vBGrouName) {
        Assert.fail("String group name objects dont match despite interning.");
    }
    tezClient.stop();
}