use of co.cask.cdap.internal.app.runtime.batch.dataset.output.ProvidedOutput in project cdap by caskdata.
the class MapReduceRuntimeService method fixOutputPermissions.
private void fixOutputPermissions(JobContext job, List<ProvidedOutput> outputs) {
Configuration jobconf = job.getConfiguration();
Set<String> outputsWithUmask = new HashSet<>();
Set<String> outputUmasks = new HashSet<>();
for (ProvidedOutput entry : outputs) {
String umask = entry.getOutputFormatConfiguration().get(HADOOP_UMASK_PROPERTY);
if (umask != null) {
outputsWithUmask.add(entry.getOutput().getAlias());
outputUmasks.add(umask);
}
}
boolean allOutputsHaveUmask = outputsWithUmask.size() == outputs.size();
boolean allOutputsAgree = outputUmasks.size() == 1;
boolean jobConfHasUmask = isProgrammaticConfig(jobconf, HADOOP_UMASK_PROPERTY);
String jobConfUmask = jobconf.get(HADOOP_UMASK_PROPERTY);
boolean mustFixUmasks = false;
if (jobConfHasUmask) {
// case 1: job conf has a programmatic umask. It prevails.
mustFixUmasks = !outputsWithUmask.isEmpty();
if (mustFixUmasks) {
LOG.info("Overriding permissions of outputs {} because a umask of {} was set programmatically in the job " + "configuration.", outputsWithUmask, jobConfUmask);
}
} else if (allOutputsHaveUmask && allOutputsAgree) {
// case 2: no programmatic umask in job conf, all outputs want the same umask: set it in job conf
String umaskToUse = outputUmasks.iterator().next();
jobconf.set(HADOOP_UMASK_PROPERTY, umaskToUse);
LOG.debug("Setting umask of {} in job configuration because all outputs {} agree on it.", umaskToUse, outputsWithUmask);
} else {
// case 3: some outputs configure a umask, but not all of them, or not all the same: use job conf default
mustFixUmasks = !outputsWithUmask.isEmpty();
if (mustFixUmasks) {
LOG.warn("Overriding permissions of outputs {} because they configure different permissions. Falling back " + "to default umask of {} in job configuration.", outputsWithUmask, jobConfUmask);
}
}
// fix all output configurations that have a umask by removing that property from their configs
if (mustFixUmasks) {
for (int i = 0; i < outputs.size(); i++) {
ProvidedOutput output = outputs.get(i);
if (outputsWithUmask.contains(output.getOutput().getAlias())) {
Map<String, String> outputConfig = new HashMap<>(output.getOutputFormatConfiguration());
outputConfig.remove(HADOOP_UMASK_PROPERTY);
outputs.set(i, new ProvidedOutput(output.getOutput(), output.getOutputFormatProvider(), output.getOutputFormatClassName(), outputConfig));
}
}
}
}
Aggregations