Examples with WebHCatJTShim org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim used on opensource projects

Example 1 with WebHCatJTShim

use of org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim in project hive by apache.

the class LaunchMapper method tryReconnectToRunningJob.

/**
   * Attempts to reconnect to an already running child job of the templeton launcher. This
   * is used in cases where the templeton launcher task has failed and is retried by the
   * MR framework. If reconnect to the child job is possible, the method will continue
   * tracking its progress until completion.
   * @return Returns true if reconnect was successful, false if not supported or
   *         no child jobs were found.
   */
private boolean tryReconnectToRunningJob(Configuration conf, Context context, LauncherDelegator.JobType jobType, String statusdir) throws IOException, InterruptedException {
    if (!reconnectToRunningJobEnabledAndSupported(conf, jobType)) {
        return false;
    }
    long startTime = getTempletonLaunchTime(conf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    WebHCatJTShim tracker = ShimLoader.getHadoopShims().getWebHCatShim(conf, ugi);
    try {
        Set<String> childJobs = tracker.getJobs(context.getJobID().toString(), startTime);
        if (childJobs.size() == 0) {
            LOG.info("No child jobs found to reconnect with");
            return false;
        }
        if (childJobs.size() > 1) {
            LOG.warn(String.format("Found more than one child job to reconnect with: %s, skipping reconnect", Arrays.toString(childJobs.toArray())));
            return false;
        }
        String childJobIdString = childJobs.iterator().next();
        org.apache.hadoop.mapred.JobID childJobId = org.apache.hadoop.mapred.JobID.forName(childJobIdString);
        LOG.info(String.format("Reconnecting to an existing job %s", childJobIdString));
        // Update job state with the childJob id
        updateJobStatePercentAndChildId(conf, context.getJobID().toString(), null, childJobIdString);
        do {
            org.apache.hadoop.mapred.JobStatus jobStatus = tracker.getJobStatus(childJobId);
            if (jobStatus.isJobComplete()) {
                LOG.info(String.format("Child job %s completed", childJobIdString));
                int exitCode = 0;
                if (jobStatus.getRunState() != org.apache.hadoop.mapred.JobStatus.SUCCEEDED) {
                    exitCode = 1;
                }
                updateJobStateToDoneAndWriteExitValue(conf, statusdir, context.getJobID().toString(), exitCode);
                break;
            }
            String percent = String.format("map %s%%, reduce %s%%", jobStatus.mapProgress() * 100, jobStatus.reduceProgress() * 100);
            updateJobStatePercentAndChildId(conf, context.getJobID().toString(), percent, null);
            LOG.info("KeepAlive Heart beat");
            context.progress();
            Thread.sleep(POLL_JOBPROGRESS_MSEC);
        } while (true);
        // Reconnect was successful
        return true;
    } catch (IOException ex) {
        LOG.error("Exception encountered in tryReconnectToRunningJob", ex);
        throw ex;
    } finally {
        tracker.close();
    }
}

Example 2 with WebHCatJTShim

use of org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim in project hive by apache.

the class LauncherDelegator method getShimLibjars.

/**
   * Dynamically determine the list of hive shim jars that need to be added
   * to the Templeton launcher job classpath.
   */
private String getShimLibjars() {
    WebHCatJTShim shim = null;
    try {
        shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, UserGroupInformation.getCurrentUser());
    } catch (IOException e) {
        throw new RuntimeException("Failed to get WebHCatShim", e);
    }
    // Besides the HiveShims jar which is Hadoop version dependent we also
    // always need to include hive shims common jars.
    Path shimCommonJar = new Path(TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN));
    Path shimCommonSecureJar = new Path(TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN));
    Path shimJar = new Path(TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN));
    return String.format("%s,%s,%s", shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString());
}

Example 3 with WebHCatJTShim

use of org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim in project hive by apache.

the class DeleteDelegator method run.

public QueueStatusBean run(String user, String id) throws NotAuthorizedException, BadParam, IOException, InterruptedException {
    UserGroupInformation ugi = UgiFactory.getUgi(user);
    WebHCatJTShim tracker = null;
    JobState state = null;
    try {
        tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi);
        JobID jobid = StatusDelegator.StringToJobID(id);
        if (jobid == null)
            throw new BadParam("Invalid jobid: " + id);
        tracker.killJob(jobid);
        state = new JobState(id, Main.getAppConfigInstance());
        List<JobState> children = state.getChildren();
        if (children != null) {
            for (JobState child : children) {
                try {
                    tracker.killJob(StatusDelegator.StringToJobID(child.getId()));
                } catch (IOException e) {
                    LOG.warn("templeton: fail to kill job " + child.getId());
                }
            }
        }
        return StatusDelegator.makeStatus(tracker, jobid, state);
    } catch (IllegalStateException e) {
        throw new BadParam(e.getMessage());
    } finally {
        if (tracker != null)
            tracker.close();
        if (state != null)
            state.close();
    }
}

Example 4 with WebHCatJTShim

use of org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim in project hive by apache.

the class StatusDelegator method getJobStatus.

public QueueStatusBean getJobStatus(String user, String id) throws NotAuthorizedException, BadParam, IOException, InterruptedException {
    WebHCatJTShim tracker = null;
    JobState state = null;
    try {
        UserGroupInformation ugi = UgiFactory.getUgi(user);
        tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi);
        JobID jobid = StatusDelegator.StringToJobID(id);
        if (jobid == null)
            throw new BadParam("Invalid jobid: " + id);
        state = new JobState(id, Main.getAppConfigInstance());
        return StatusDelegator.makeStatus(tracker, jobid, state);
    } catch (IllegalStateException e) {
        throw new BadParam(e.getMessage());
    } finally {
        if (tracker != null)
            tracker.close();
        if (state != null)
            state.close();
    }
}

Example 5 with WebHCatJTShim

use of org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim in project hive by apache.

the class LaunchMapper method killLauncherChildJobs.

/**
   * Kills child jobs of this launcher that have been tagged with this job's ID.
   */
private void killLauncherChildJobs(Configuration conf, String jobId) throws IOException {
    // Extract the launcher job submit/start time and use that to scope down
    // the search interval when we look for child jobs
    long startTime = getTempletonLaunchTime(conf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    WebHCatJTShim tracker = ShimLoader.getHadoopShims().getWebHCatShim(conf, ugi);
    try {
        tracker.killJobs(jobId, startTime);
    } finally {
        tracker.close();
    }
}