Examples with SimplePigStats org.apache.pig.tools.pigstats.mapreduce.SimplePigStats used on opensource projects

Example 1 with SimplePigStats

use of org.apache.pig.tools.pigstats.mapreduce.SimplePigStats in project zeppelin by apache.

the class PigUtils method extractJobIdsFromSimplePigStats.

public static List<String> extractJobIdsFromSimplePigStats(SimplePigStats stat) {
    List<String> jobIds = new ArrayList<>();
    try {
        Field jobPlanField = PigStats.class.getDeclaredField("jobPlan");
        jobPlanField.setAccessible(true);
        PigStats.JobGraph jobPlan = (PigStats.JobGraph) jobPlanField.get(stat);
        List<JobStats> arr = jobPlan.getJobList();
        for (JobStats js : arr) {
            jobIds.add(js.getJobId());
        }
        return jobIds;
    } catch (Exception e) {
        LOGGER.error("Can not extract jobIds from SimpelPigStats", e);
        throw new RuntimeException("Can not extract jobIds from SimpelPigStats", e);
    }
}

Example 2 with SimplePigStats

use of org.apache.pig.tools.pigstats.mapreduce.SimplePigStats in project zeppelin by apache.

the class PigUtils method extractFromSimplePigStats.

public static String extractFromSimplePigStats(SimplePigStats stats) {
    try {
        Field userIdField = PigStats.class.getDeclaredField("userId");
        userIdField.setAccessible(true);
        String userId = (String) (userIdField.get(stats));
        Field startTimeField = PigStats.class.getDeclaredField("startTime");
        startTimeField.setAccessible(true);
        long startTime = (Long) (startTimeField.get(stats));
        Field endTimeField = PigStats.class.getDeclaredField("endTime");
        endTimeField.setAccessible(true);
        long endTime = (Long) (endTimeField.get(stats));
        if (stats.getReturnCode() == PigRunner.ReturnCode.UNKNOWN) {
            LOGGER.warn("unknown return code, can't display the results");
            return null;
        }
        if (stats.getPigContext() == null) {
            LOGGER.warn("unknown exec type, don't display the results");
            return null;
        }
        SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
        StringBuilder sb = new StringBuilder();
        sb.append("\nHadoopVersion\tPigVersion\tUserId\tStartedAt\tFinishedAt\tFeatures\n");
        sb.append(stats.getHadoopVersion()).append("\t").append(stats.getPigVersion()).append("\t").append(userId).append("\t").append(sdf.format(new Date(startTime))).append("\t").append(sdf.format(new Date(endTime))).append("\t").append(stats.getFeatures()).append("\n");
        sb.append("\n");
        if (stats.getReturnCode() == PigRunner.ReturnCode.SUCCESS) {
            sb.append("Success!\n");
        } else if (stats.getReturnCode() == PigRunner.ReturnCode.PARTIAL_FAILURE) {
            sb.append("Some jobs have failed! Stop running all dependent jobs\n");
        } else {
            sb.append("Failed!\n");
        }
        sb.append("\n");
        Field jobPlanField = PigStats.class.getDeclaredField("jobPlan");
        jobPlanField.setAccessible(true);
        PigStats.JobGraph jobPlan = (PigStats.JobGraph) jobPlanField.get(stats);
        if (stats.getReturnCode() == PigRunner.ReturnCode.SUCCESS || stats.getReturnCode() == PigRunner.ReturnCode.PARTIAL_FAILURE) {
            sb.append("Job Stats (time in seconds):\n");
            sb.append(MRJobStats.SUCCESS_HEADER).append("\n");
            List<JobStats> arr = jobPlan.getSuccessfulJobs();
            for (JobStats js : arr) {
                sb.append(js.getDisplayString());
            }
            sb.append("\n");
        }
        if (stats.getReturnCode() == PigRunner.ReturnCode.FAILURE || stats.getReturnCode() == PigRunner.ReturnCode.PARTIAL_FAILURE) {
            sb.append("Failed Jobs:\n");
            sb.append(MRJobStats.FAILURE_HEADER).append("\n");
            List<JobStats> arr = jobPlan.getFailedJobs();
            for (JobStats js : arr) {
                sb.append(js.getDisplayString());
            }
            sb.append("\n");
        }
        sb.append("Input(s):\n");
        for (InputStats is : stats.getInputStats()) {
            sb.append(is.getDisplayString());
        }
        sb.append("\n");
        sb.append("Output(s):\n");
        for (OutputStats ds : stats.getOutputStats()) {
            sb.append(ds.getDisplayString());
        }
        sb.append("\nCounters:\n");
        sb.append("Total records written : " + stats.getRecordWritten()).append("\n");
        sb.append("Total bytes written : " + stats.getBytesWritten()).append("\n");
        sb.append("Spillable Memory Manager spill count : " + stats.getSMMSpillCount()).append("\n");
        sb.append("Total bags proactively spilled: " + stats.getProactiveSpillCountObjects()).append("\n");
        sb.append("Total records proactively spilled: " + stats.getProactiveSpillCountRecords()).append("\n");
        sb.append("\nJob DAG:\n").append(jobPlan.toString());
        return "Script Statistics: \n" + sb.toString();
    } catch (Exception e) {
        LOGGER.error("Can not extract message from SimplePigStats", e);
        return "Can not extract message from SimpelPigStats," + ExceptionUtils.getStackTrace(e);
    }
}