Examples with VertexStatus org.apache.tez.dag.api.client.VertexStatus used on opensource projects

Example 6 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project hive by apache.

the class TezTask method collectCommitInformation.

private void collectCommitInformation(TezWork work) throws IOException, TezException {
    for (BaseWork w : work.getAllWork()) {
        JobConf jobConf = workToConf.get(w);
        Vertex vertex = workToVertex.get(w);
        boolean hasIcebergCommitter = Optional.ofNullable(jobConf).map(JobConf::getOutputCommitter).map(Object::getClass).map(Class::getName).filter(name -> name.endsWith("HiveIcebergNoJobCommitter")).isPresent();
        // we should only consider jobs with Iceberg output committer and a data sink
        if (hasIcebergCommitter && !vertex.getDataSinks().isEmpty()) {
            VertexStatus status = dagClient.getVertexStatus(vertex.getName(), EnumSet.of(StatusGetOpts.GET_COUNTERS));
            String[] jobIdParts = status.getId().split("_");
            // status.getId() returns something like: vertex_1617722404520_0001_1_00
            // this should be transformed to a parsable JobID: job_16177224045200_0001
            int vertexId = Integer.parseInt(jobIdParts[jobIdParts.length - 1]);
            String jobId = String.format(JOB_ID_TEMPLATE, jobIdParts[1], vertexId, jobIdParts[2]);
            List<String> tables = new ArrayList<>();
            Map<String, String> icebergProperties = new HashMap<>();
            for (Map.Entry<String, String> entry : jobConf) {
                if (entry.getKey().startsWith(ICEBERG_SERIALIZED_TABLE_PREFIX)) {
                    // get all target tables this vertex wrote to
                    tables.add(entry.getKey().substring(ICEBERG_SERIALIZED_TABLE_PREFIX.length()));
                } else if (entry.getKey().startsWith(ICEBERG_PROPERTY_PREFIX)) {
                    // find iceberg props in jobConf as they can be needed, but not available, during job commit
                    icebergProperties.put(entry.getKey(), entry.getValue());
                }
            }
            // save information for each target table
            tables.forEach(table -> SessionStateUtil.addCommitInfo(jobConf, table, jobId, status.getProgress().getSucceededTaskCount(), icebergProperties));
        }
    }
}

Example 7 with VertexStatus

use of org.apache.tez.dag.api.client.VertexStatus in project hive by apache.

the class DAGSummary method print.

@Override
public void print(SessionState.LogHelper console) {
    console.printInfo("Task Execution Summary");
    /* If the counters are missing there is no point trying to print progress */
    if (hiveCounters == null) {
        return;
    }
    /* Print the per Vertex summary */
    printHeader(console);
    SortedSet<String> keys = new TreeSet<>(progressMap.keySet());
    Set<StatusGetOpts> statusOptions = new HashSet<>(1);
    statusOptions.add(StatusGetOpts.GET_COUNTERS);
    for (String vertexName : keys) {
        Progress progress = progressMap.get(vertexName);
        if (progress == null)
            continue;
        VertexStatus vertexStatus = vertexStatus(statusOptions, vertexName);
        if (vertexStatus == null) {
            continue;
        }
        console.printInfo(vertexSummary(vertexName, progress, vertexStatus));
    }
    console.printInfo(FILE_HEADER_SEPARATOR);
}