Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 21 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class ExampleDriver method printDAGStatus.

public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, boolean displayDAGCounters, boolean displayVertexCounters) throws IOException, TezException {
    Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
    DAGStatus dagStatus = dagClient.getDAGStatus((displayDAGCounters ? opts : null));
    Progress progress = dagStatus.getDAGProgress();
    double vProgressFloat = 0.0f;
    if (progress != null) {
        System.out.println("");
        System.out.println("DAG: State: " + dagStatus.getState() + " Progress: " + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) : formatter.format((double) (progress.getSucceededTaskCount()) / progress.getTotalTaskCount())));
        for (String vertexName : vertexNames) {
            VertexStatus vStatus = dagClient.getVertexStatus(vertexName, (displayVertexCounters ? opts : null));
            if (vStatus == null) {
                System.out.println("Could not retrieve status for vertex: " + vertexName);
                continue;
            }
            Progress vProgress = vStatus.getProgress();
            if (vProgress != null) {
                vProgressFloat = 0.0f;
                if (vProgress.getTotalTaskCount() == 0) {
                    vProgressFloat = 1.0f;
                } else if (vProgress.getTotalTaskCount() > 0) {
                    vProgressFloat = (double) vProgress.getSucceededTaskCount() / vProgress.getTotalTaskCount();
                }
                System.out.println("VertexStatus:" + " VertexName: " + (vertexName.equals("ivertex1") ? "intermediate-reducer" : vertexName) + " Progress: " + formatter.format(vProgressFloat));
            }
            if (displayVertexCounters) {
                TezCounters counters = vStatus.getVertexCounters();
                if (counters != null) {
                    System.out.println("Vertex Counters for " + vertexName + ": " + counters);
                }
            }
        }
    }
    if (displayDAGCounters) {
        TezCounters counters = dagStatus.getDAGCounters();
        if (counters != null) {
            System.out.println("DAG Counters: " + counters);
        }
    }
}

Example 22 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class DAGStatus method hashCode.

@Override
public int hashCode() {
    // Source explicitly exclude from hashCode
    final int prime = 44017;
    int result = 1;
    result = prime + getState().hashCode();
    List<String> diagnostics = getDiagnostics();
    Progress dagProgress = getDAGProgress();
    Map<String, Progress> vProgress = getVertexProgress();
    TezCounters counters = getDAGCounters();
    result = prime * result + ((diagnostics == null) ? 0 : diagnostics.hashCode());
    result = prime * result + ((dagProgress == null) ? 0 : dagProgress.hashCode());
    result = prime * result + ((vProgress == null) ? 0 : vProgress.hashCode());
    result = prime * result + ((counters == null) ? 0 : counters.hashCode());
    return result;
}

Example 23 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class VertexStatus method hashCode.

@Override
public int hashCode() {
    final int prime = 46021;
    int result = 1;
    result = prime + getState().hashCode();
    List<String> diagnostics = getDiagnostics();
    Progress vProgress = getProgress();
    TezCounters counters = getVertexCounters();
    result = prime * result + ((diagnostics == null) ? 0 : diagnostics.hashCode());
    result = prime * result + ((vProgress == null) ? 0 : vProgress.hashCode());
    result = prime * result + ((counters == null) ? 0 : counters.hashCode());
    return result;
}

Example 24 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestShuffleManager method createInputContext.

private InputContext createInputContext() throws IOException {
    DataOutputBuffer port_dob = new DataOutputBuffer();
    port_dob.writeInt(PORT);
    final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());
    port_dob.close();
    ExecutionContext executionContext = mock(ExecutionContext.class);
    doReturn(FETCHER_HOST).when(executionContext).getHostName();
    InputContext inputContext = mock(InputContext.class);
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("sourceVertex").when(inputContext).getSourceVertexName();
    doReturn("taskVertex").when(inputContext).getTaskVertexName();
    doReturn(shuffleMetaData).when(inputContext).getServiceProviderMetaData(conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
    doReturn(executionContext).when(inputContext).getExecutionContext();
    when(inputContext.createTezFrameworkExecutorService(anyInt(), anyString())).thenAnswer(new Answer<ExecutorService>() {

        @Override
        public ExecutorService answer(InvocationOnMock invocation) throws Throwable {
            return sharedExecutor.createExecutorService(invocation.getArgumentAt(0, Integer.class), invocation.getArgumentAt(1, String.class));
        }
    });
    return inputContext;
}

Example 25 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestFetcher method createMockInputContext.

private InputContext createMockInputContext() {
    InputContext inputContext = mock(InputContext.class);
    doReturn(APP_ID).when(inputContext).getApplicationId();
    doReturn(DAG_ID).when(inputContext).getDagIdentifier();
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("src vertex").when(inputContext).getSourceVertexName();
    doReturn("task_Vertex").when(inputContext).getTaskVertexName();
    return inputContext;
}