Search in sources :

Example 1 with DelegationTokenIdentifier

use of org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier in project hive by apache.

the class TempletonControllerJob method run.

/**
   * Enqueue the job and print out the job id for later collection.
   * @see org.apache.hive.hcatalog.templeton.CompleteDelegator
   */
@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, TException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Preparing to submit job: " + Arrays.toString(args));
    }
    Configuration conf = getConf();
    conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args));
    String memoryMb = appConf.mapperMemoryMb();
    if (memoryMb != null && memoryMb.length() != 0) {
        conf.set(AppConfig.HADOOP_MAP_MEMORY_MB, memoryMb);
    }
    String amMemoryMB = appConf.amMemoryMb();
    if (amMemoryMB != null && !amMemoryMB.isEmpty()) {
        conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB);
    }
    String amJavaOpts = appConf.controllerAMChildOpts();
    if (amJavaOpts != null && !amJavaOpts.isEmpty()) {
        conf.set(AppConfig.HADOOP_MR_AM_JAVA_OPTS, amJavaOpts);
    }
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    conf.set("user.name", user);
    job = new Job(conf);
    job.setJarByClass(LaunchMapper.class);
    job.setJobName(TempletonControllerJob.class.getSimpleName());
    job.setMapperClass(LaunchMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setInputFormatClass(SingleInputFormat.class);
    NullOutputFormat<NullWritable, NullWritable> of = new NullOutputFormat<NullWritable, NullWritable>();
    job.setOutputFormatClass(of.getClass());
    job.setNumReduceTasks(0);
    JobClient jc = new JobClient(new JobConf(job.getConfiguration()));
    if (UserGroupInformation.isSecurityEnabled()) {
        Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
        job.getCredentials().addToken(new Text("mr token"), mrdt);
    }
    String metastoreTokenStrForm = addHMSToken(job, user);
    job.submit();
    JobID submittedJobId = job.getJobID();
    if (metastoreTokenStrForm != null) {
        //so that it can be cancelled later from CompleteDelegator
        DelegationTokenCache.getStringFormTokenCache().storeDelegationToken(submittedJobId.toString(), metastoreTokenStrForm);
        LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() + " user=" + user);
    }
    return 0;
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) DelegationTokenIdentifier(org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier) Text(org.apache.hadoop.io.Text) NullWritable(org.apache.hadoop.io.NullWritable) JobClient(org.apache.hadoop.mapred.JobClient) Job(org.apache.hadoop.mapreduce.Job) JobConf(org.apache.hadoop.mapred.JobConf) NullOutputFormat(org.apache.hadoop.mapreduce.lib.output.NullOutputFormat) JobID(org.apache.hadoop.mapreduce.JobID)

Aggregations

Configuration (org.apache.hadoop.conf.Configuration)1 NullWritable (org.apache.hadoop.io.NullWritable)1 Text (org.apache.hadoop.io.Text)1 JobClient (org.apache.hadoop.mapred.JobClient)1 JobConf (org.apache.hadoop.mapred.JobConf)1 Job (org.apache.hadoop.mapreduce.Job)1 JobID (org.apache.hadoop.mapreduce.JobID)1 NullOutputFormat (org.apache.hadoop.mapreduce.lib.output.NullOutputFormat)1 DelegationTokenIdentifier (org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier)1