Examples with VertexStatus org.apache.tez.dag.api.client.VertexStatus used on opensource projects

Example 1 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project tez by apache.

the class ExampleDriver method printDAGStatus.

public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, boolean displayDAGCounters, boolean displayVertexCounters) throws IOException, TezException {
    Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
    DAGStatus dagStatus = dagClient.getDAGStatus((displayDAGCounters ? opts : null));
    Progress progress = dagStatus.getDAGProgress();
    double vProgressFloat = 0.0f;
    if (progress != null) {
        System.out.println("");
        System.out.println("DAG: State: " + dagStatus.getState() + " Progress: " + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) : formatter.format((double) (progress.getSucceededTaskCount()) / progress.getTotalTaskCount())));
        for (String vertexName : vertexNames) {
            VertexStatus vStatus = dagClient.getVertexStatus(vertexName, (displayVertexCounters ? opts : null));
            if (vStatus == null) {
                System.out.println("Could not retrieve status for vertex: " + vertexName);
                continue;
            }
            Progress vProgress = vStatus.getProgress();
            if (vProgress != null) {
                vProgressFloat = 0.0f;
                if (vProgress.getTotalTaskCount() == 0) {
                    vProgressFloat = 1.0f;
                } else if (vProgress.getTotalTaskCount() > 0) {
                    vProgressFloat = (double) vProgress.getSucceededTaskCount() / vProgress.getTotalTaskCount();
                }
                System.out.println("VertexStatus:" + " VertexName: " + (vertexName.equals("ivertex1") ? "intermediate-reducer" : vertexName) + " Progress: " + formatter.format(vProgressFloat));
            }
            if (displayVertexCounters) {
                TezCounters counters = vStatus.getVertexCounters();
                if (counters != null) {
                    System.out.println("Vertex Counters for " + vertexName + ": " + counters);
                }
            }
        }
    }
    if (displayDAGCounters) {
        TezCounters counters = dagStatus.getDAGCounters();
        if (counters != null) {
            System.out.println("DAG Counters: " + counters);
        }
    }
}

Example 2 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project hive by apache.

the class DAGSummary method hiveInputRecordsFromTezCounters.

private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) {
    // Get the counters for the input vertex.
    Set<StatusGetOpts> statusOptions = Collections.singleton(StatusGetOpts.GET_COUNTERS);
    VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName);
    final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters();
    // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS
    String groupName = formattedName("TaskCounter", inputVertexName, vertexName);
    String counterName = "OUTPUT_RECORDS";
    // Do not create counter if it does not exist -
    // instead fall back to default behavior for determining input records.
    TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false);
    if (tezCounter == null) {
        return -1;
    } else {
        return tezCounter.getValue();
    }
}

Example 3 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project tez by apache.

the class TestTezJobs method testPerIOCounterAggregation.

@Test(timeout = 60000)
public void testPerIOCounterAggregation() throws Exception {
    String baseDir = "/tmp/perIOCounterAgg/";
    Path inPath1 = new Path(baseDir + "inPath1");
    Path inPath2 = new Path(baseDir + "inPath2");
    Path outPath = new Path(baseDir + "outPath");
    final Set<String> expectedResults = generateSortMergeJoinInput(inPath1, inPath2);
    Path stagingDirPath = new Path("/tmp/tez-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    TezConfiguration conf = new TezConfiguration(mrrTezCluster.getConfig());
    conf.setBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, true);
    TezClient tezClient = TezClient.create(SortMergeJoinHelper.class.getSimpleName(), conf);
    tezClient.start();
    SortMergeJoinHelper sortMergeJoinHelper = new SortMergeJoinHelper(tezClient);
    sortMergeJoinHelper.setConf(conf);
    String[] args = new String[] { "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() };
    assertEquals(0, sortMergeJoinHelper.run(conf, args, tezClient));
    verifySortMergeJoinInput(outPath, expectedResults);
    String joinerVertexName = "joiner";
    String input1Name = "input1";
    String input2Name = "input2";
    String joinOutputName = "joinOutput";
    Set<StatusGetOpts> statusOpts = new HashSet<StatusGetOpts>();
    statusOpts.add(StatusGetOpts.GET_COUNTERS);
    VertexStatus joinerVertexStatus = sortMergeJoinHelper.dagClient.getVertexStatus(joinerVertexName, statusOpts);
    final TezCounters joinerCounters = joinerVertexStatus.getVertexCounters();
    final CounterGroup aggregatedGroup = joinerCounters.getGroup(TaskCounter.class.getCanonicalName());
    final CounterGroup input1Group = joinerCounters.getGroup(TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input1Name);
    final CounterGroup input2Group = joinerCounters.getGroup(TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input2Name);
    assertTrue("aggregated counter group cannot be empty", aggregatedGroup.size() > 0);
    assertTrue("per io group for input1 cannot be empty", input1Group.size() > 0);
    assertTrue("per io group for input1 cannot be empty", input2Group.size() > 0);
    List<TaskCounter> countersToVerifyAgg = Arrays.asList(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ, TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN, TaskCounter.COMBINE_INPUT_RECORDS, TaskCounter.MERGED_MAP_OUTPUTS, TaskCounter.NUM_DISK_TO_DISK_MERGES, TaskCounter.NUM_FAILED_SHUFFLE_INPUTS, TaskCounter.NUM_MEM_TO_DISK_MERGES, TaskCounter.NUM_SHUFFLED_INPUTS, TaskCounter.NUM_SKIPPED_INPUTS, TaskCounter.REDUCE_INPUT_GROUPS, TaskCounter.REDUCE_INPUT_RECORDS, TaskCounter.SHUFFLE_BYTES, TaskCounter.SHUFFLE_BYTES_DECOMPRESSED, TaskCounter.SHUFFLE_BYTES_DISK_DIRECT, TaskCounter.SHUFFLE_BYTES_TO_DISK, TaskCounter.SHUFFLE_BYTES_TO_MEM, TaskCounter.SPILLED_RECORDS);
    int nonZeroCounters = 0;
    // verify that the sum of the counter values for edges add up to the aggregated counter value.
    for (TaskCounter c : countersToVerifyAgg) {
        TezCounter aggregatedCounter = aggregatedGroup.findCounter(c.name(), false);
        TezCounter input1Counter = input1Group.findCounter(c.name(), false);
        TezCounter input2Counter = input2Group.findCounter(c.name(), false);
        assertNotNull("aggregated counter cannot be null " + c.name(), aggregatedCounter);
        assertNotNull("input1 counter cannot be null " + c.name(), input1Counter);
        assertNotNull("input2 counter cannot be null " + c.name(), input2Counter);
        assertEquals("aggregated counter does not match sum of input counters " + c.name(), aggregatedCounter.getValue(), input1Counter.getValue() + input2Counter.getValue());
        if (aggregatedCounter.getValue() > 0) {
            nonZeroCounters++;
        }
    }
    // ensure that at least one of the counters tested above were non-zero.
    assertTrue("At least one of the counter should be non-zero. invalid test ", nonZeroCounters > 0);
    CounterGroup joinerOutputGroup = joinerCounters.getGroup(TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_OUTPUT_" + joinOutputName);
    String outputCounterName = TaskCounter.OUTPUT_RECORDS.name();
    TezCounter aggregateCounter = aggregatedGroup.findCounter(outputCounterName, false);
    TezCounter joinerOutputCounter = joinerOutputGroup.findCounter(outputCounterName, false);
    assertNotNull("aggregated counter cannot be null " + outputCounterName, aggregateCounter);
    assertNotNull("output counter cannot be null " + outputCounterName, joinerOutputCounter);
    assertTrue("counter value is zero. test is invalid", aggregateCounter.getValue() > 0);
    assertEquals("aggregated counter does not match sum of output counters " + outputCounterName, aggregateCounter.getValue(), joinerOutputCounter.getValue());
}

Example 4 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project tez by apache.

the class TestDAGClient method testVertexStatus.

@Test(timeout = 5000)
public void testVertexStatus() throws Exception {
    VertexStatus resultVertexStatus = dagClient.getVertexStatus("v1", null);
    verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder().setDagId(dagIdStr).setVertexName("v1").build());
    assertEquals(new VertexStatus(vertexStatusProtoWithoutCounters), resultVertexStatus);
    System.out.println("VertexWithoutCounter:" + resultVertexStatus);
    resultVertexStatus = dagClient.getVertexStatus("v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder().setDagId(dagIdStr).setVertexName("v1").addStatusOptions(StatusGetOptsProto.GET_COUNTERS).build());
    assertEquals(new VertexStatus(vertexStatusProtoWithCounters), resultVertexStatus);
    System.out.println("VertexWithCounter:" + resultVertexStatus);
}

Example 5 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project tez by apache.

the class DAGClientAMProtocolBlockingPBServerImpl method getVertexStatus.

@Override
public GetVertexStatusResponseProto getVertexStatus(RpcController controller, GetVertexStatusRequestProto request) throws ServiceException {
    UserGroupInformation user = getRPCUser();
    try {
        String dagId = request.getDagId();
        if (!real.getACLManager(dagId).checkDAGViewAccess(user)) {
            throw new AccessControlException("User " + user + " cannot perform DAG view operation");
        }
        real.updateLastHeartbeatTime();
        String vertexName = request.getVertexName();
        VertexStatus status = real.getVertexStatus(dagId, vertexName, DagTypeConverters.convertStatusGetOptsFromProto(request.getStatusOptionsList()));
        assert status instanceof VertexStatusBuilder;
        VertexStatusBuilder builder = (VertexStatusBuilder) status;
        return GetVertexStatusResponseProto.newBuilder().setVertexStatus(builder.getProto()).build();
    } catch (TezException e) {
        throw wrapException(e);
    }
}