use of org.apache.hadoop.hive.ql.util.ResourceDownloader in project hive by apache.
the class LlapServiceDriver method downloadPermanentFunctions.
private Set<String> downloadPermanentFunctions(Configuration conf, Path udfDir) throws HiveException, URISyntaxException, IOException {
Map<String, String> udfs = new HashMap<String, String>();
HiveConf hiveConf = new HiveConf();
// disable expensive operations on the metastore
hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_INIT_METADATA_COUNT_ENABLED, false);
hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, false);
// performance problem: ObjectStore does its own new HiveConf()
Hive hive = Hive.getWithFastCheck(hiveConf, false);
ResourceDownloader resourceDownloader = new ResourceDownloader(conf, udfDir.toUri().normalize().getPath());
List<Function> fns = hive.getAllFunctions();
Set<URI> srcUris = new HashSet<>();
for (Function fn : fns) {
String fqfn = fn.getDbName() + "." + fn.getFunctionName();
if (udfs.containsKey(fn.getClassName())) {
LOG.warn("Duplicate function names found for " + fn.getClassName() + " with " + fqfn + " and " + udfs.get(fn.getClassName()));
}
udfs.put(fn.getClassName(), fqfn);
List<ResourceUri> resources = fn.getResourceUris();
if (resources == null || resources.isEmpty()) {
LOG.warn("Missing resources for " + fqfn);
continue;
}
for (ResourceUri resource : resources) {
srcUris.add(ResourceDownloader.createURI(resource.getUri()));
}
}
for (URI srcUri : srcUris) {
List<URI> localUris = resourceDownloader.downloadExternal(srcUri, null, false);
for (URI dst : localUris) {
LOG.warn("Downloaded " + dst + " from " + srcUri);
}
}
return udfs.keySet();
}
use of org.apache.hadoop.hive.ql.util.ResourceDownloader in project hive by apache.
the class FunctionLocalizer method init.
public void init() throws IOException {
if (localDir.exists()) {
// TODO: We don't want some random jars of unknown provenance sitting around. Or do we care?
// Ideally, we should try to reuse jars and verify using some checksum.
FileUtils.deleteDirectory(localDir);
}
this.resourceDownloader = new ResourceDownloader(conf, localDir.getAbsolutePath());
workThread.start();
}
Aggregations