Examples with TezDAGStats org.apache.pig.tools.pigstats.tez.TezDAGStats used on opensource projects

Example 1 with TezDAGStats

use of org.apache.pig.tools.pigstats.tez.TezDAGStats in project zeppelin by apache.

the class PigUtils method extractFromTezPigStats.

private static String extractFromTezPigStats(TezPigScriptStats stats) {
    try {
        if (stats.getReturnCode() == PigRunner.ReturnCode.UNKNOWN) {
            LOGGER.warn("unknown return code, can't display the results");
            return null;
        }
        if (stats.getPigContext() == null) {
            LOGGER.warn("unknown exec type, don't display the results");
            return null;
        }
        Field userIdField = PigStats.class.getDeclaredField("userId");
        userIdField.setAccessible(true);
        String userId = (String) (userIdField.get(stats));
        Field startTimeField = PigStats.class.getDeclaredField("startTime");
        startTimeField.setAccessible(true);
        long startTime = (Long) (startTimeField.get(stats));
        Field endTimeField = PigStats.class.getDeclaredField("endTime");
        endTimeField.setAccessible(true);
        long endTime = (Long) (endTimeField.get(stats));
        SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
        StringBuilder sb = new StringBuilder();
        sb.append("\n");
        sb.append(String.format("%1$20s: %2$-100s%n", "HadoopVersion", stats.getHadoopVersion()));
        sb.append(String.format("%1$20s: %2$-100s%n", "PigVersion", stats.getPigVersion()));
        sb.append(String.format("%1$20s: %2$-100s%n", "TezVersion", TezExecType.getTezVersion()));
        sb.append(String.format("%1$20s: %2$-100s%n", "UserId", userId));
        sb.append(String.format("%1$20s: %2$-100s%n", "FileName", stats.getFileName()));
        sb.append(String.format("%1$20s: %2$-100s%n", "StartedAt", sdf.format(new Date(startTime))));
        sb.append(String.format("%1$20s: %2$-100s%n", "FinishedAt", sdf.format(new Date(endTime))));
        sb.append(String.format("%1$20s: %2$-100s%n", "Features", stats.getFeatures()));
        sb.append("\n");
        if (stats.getReturnCode() == PigRunner.ReturnCode.SUCCESS) {
            sb.append("Success!\n");
        } else if (stats.getReturnCode() == PigRunner.ReturnCode.PARTIAL_FAILURE) {
            sb.append("Some tasks have failed! Stop running all dependent tasks\n");
        } else {
            sb.append("Failed!\n");
        }
        sb.append("\n");
        // Print diagnostic info in case of failure
        if (stats.getReturnCode() == PigRunner.ReturnCode.FAILURE || stats.getReturnCode() == PigRunner.ReturnCode.PARTIAL_FAILURE) {
            if (stats.getErrorMessage() != null) {
                String[] lines = stats.getErrorMessage().split("\n");
                for (int i = 0; i < lines.length; i++) {
                    String s = lines[i].trim();
                    if (i == 0 || !org.apache.commons.lang.StringUtils.isEmpty(s)) {
                        sb.append(String.format("%1$20s: %2$-100s%n", i == 0 ? "ErrorMessage" : "", s));
                    }
                }
                sb.append("\n");
            }
        }
        Field tezDAGStatsMapField = TezPigScriptStats.class.getDeclaredField("tezDAGStatsMap");
        tezDAGStatsMapField.setAccessible(true);
        Map<String, TezDAGStats> tezDAGStatsMap = (Map<String, TezDAGStats>) tezDAGStatsMapField.get(stats);
        int count = 0;
        for (TezDAGStats dagStats : tezDAGStatsMap.values()) {
            sb.append("\n");
            sb.append("DAG " + count++ + ":\n");
            sb.append(dagStats.getDisplayString());
            sb.append("\n");
        }
        sb.append("Input(s):\n");
        for (InputStats is : stats.getInputStats()) {
            sb.append(is.getDisplayString().trim()).append("\n");
        }
        sb.append("\n");
        sb.append("Output(s):\n");
        for (OutputStats os : stats.getOutputStats()) {
            sb.append(os.getDisplayString().trim()).append("\n");
        }
        return "Script Statistics:\n" + sb.toString();
    } catch (Exception e) {
        LOGGER.error("Can not extract message from SimplePigStats", e);
        return "Can not extract message from SimpelPigStats," + ExceptionUtils.getStackTrace(e);
    }
}

Example 2 with TezDAGStats

use of org.apache.pig.tools.pigstats.tez.TezDAGStats in project zeppelin by apache.

the class PigUtils method extractJobIdsFromTezPigStats.

public static List<String> extractJobIdsFromTezPigStats(TezPigScriptStats stat) {
    List<String> jobIds = new ArrayList<>();
    try {
        Field tezDAGStatsMapField = TezPigScriptStats.class.getDeclaredField("tezDAGStatsMap");
        tezDAGStatsMapField.setAccessible(true);
        Map<String, TezDAGStats> tezDAGStatsMap = (Map<String, TezDAGStats>) tezDAGStatsMapField.get(stat);
        for (TezDAGStats dagStats : tezDAGStatsMap.values()) {
            LOGGER.debug("Tez JobId:" + dagStats.getJobId());
            jobIds.add(dagStats.getJobId());
        }
        return jobIds;
    } catch (Exception e) {
        LOGGER.error("Can not extract jobIds from TezPigScriptStats", e);
        throw new RuntimeException("Can not extract jobIds from TezPigScriptStats", e);
    }
}