use of org.apache.drill.exec.expr.fn.registry.JarScan in project drill by apache.
the class FunctionImplementationRegistry method syncWithRemoteRegistry.
/**
* Purpose of this method is to synchronize remote and local function registries if needed
* and to inform if function registry was changed after given version.
*
* To make synchronization as much light-weigh as possible, first only versions of both registries are checked
* without any locking. If synchronization is needed, enters synchronized block to prevent others loading the same jars.
* The need of synchronization is checked again (double-check lock) before comparing jars.
* If any missing jars are found, they are downloaded to local udf area, each is wrapped into {@link JarScan}.
* Once jar download is finished, all missing jars are registered in one batch.
* In case if any errors during jars download / registration, these errors are logged.
*
* During registration local function registry is updated with remote function registry version it is synced with.
* When at least one jar of the missing jars failed to download / register,
* local function registry version are not updated but jars that where successfully downloaded / registered
* are added to local function registry.
*
* If synchronization between remote and local function registry was not needed,
* checks if given registry version matches latest sync version
* to inform if function registry was changed after given version.
*
* @param version remote function registry local function registry was based on
* @return true if remote and local function registries were synchronized after given version
*/
@SuppressWarnings("resource")
public boolean syncWithRemoteRegistry(long version) {
if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localFunctionRegistry.getVersion())) {
synchronized (this) {
long localRegistryVersion = localFunctionRegistry.getVersion();
if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localRegistryVersion)) {
DataChangeVersion remoteVersion = new DataChangeVersion();
List<String> missingJars = getMissingJars(this.remoteFunctionRegistry, localFunctionRegistry, remoteVersion);
List<JarScan> jars = Lists.newArrayList();
if (!missingJars.isEmpty()) {
logger.info("Starting dynamic UDFs lazy-init process.\n" + "The following jars are going to be downloaded and registered locally: " + missingJars);
for (String jarName : missingJars) {
Path binary = null;
Path source = null;
URLClassLoader classLoader = null;
try {
binary = copyJarToLocal(jarName, this.remoteFunctionRegistry);
source = copyJarToLocal(JarUtil.getSourceName(jarName), this.remoteFunctionRegistry);
URL[] urls = { binary.toUri().toURL(), source.toUri().toURL() };
classLoader = new URLClassLoader(urls);
ScanResult scanResult = scan(classLoader, binary, urls);
localFunctionRegistry.validate(jarName, scanResult);
jars.add(new JarScan(jarName, scanResult, classLoader));
} catch (Exception e) {
deleteQuietlyLocalJar(binary);
deleteQuietlyLocalJar(source);
if (classLoader != null) {
try {
classLoader.close();
} catch (Exception ex) {
logger.warn("Problem during closing class loader for {}", jarName, e);
}
}
logger.error("Problem during remote functions load from {}", jarName, e);
}
}
}
long latestRegistryVersion = jars.size() != missingJars.size() ? localRegistryVersion : remoteVersion.getVersion();
localFunctionRegistry.register(jars, latestRegistryVersion);
return true;
}
}
}
return version != localFunctionRegistry.getVersion();
}
use of org.apache.drill.exec.expr.fn.registry.JarScan in project drill by axbaretto.
the class FunctionImplementationRegistry method syncWithRemoteRegistry.
/**
* Purpose of this method is to synchronize remote and local function registries if needed
* and to inform if function registry was changed after given version.
*
* To make synchronization as much light-weigh as possible, first only versions of both registries are checked
* without any locking. If synchronization is needed, enters synchronized block to prevent others loading the same jars.
* The need of synchronization is checked again (double-check lock) before comparing jars.
* If any missing jars are found, they are downloaded to local udf area, each is wrapped into {@link JarScan}.
* Once jar download is finished, all missing jars are registered in one batch.
* In case if any errors during jars download / registration, these errors are logged.
*
* During registration local function registry is updated with remote function registry version it is synced with.
* When at least one jar of the missing jars failed to download / register,
* local function registry version are not updated but jars that where successfully downloaded / registered
* are added to local function registry.
*
* If synchronization between remote and local function registry was not needed,
* checks if given registry version matches latest sync version
* to inform if function registry was changed after given version.
*
* @param version remote function registry local function registry was based on
* @return true if remote and local function registries were synchronized after given version
*/
@SuppressWarnings("resource")
public boolean syncWithRemoteRegistry(long version) {
// not exist for some JMockit-based unit tests.
if (isRegistrySyncNeeded()) {
synchronized (this) {
long localRegistryVersion = localFunctionRegistry.getVersion();
if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localRegistryVersion)) {
DataChangeVersion remoteVersion = new DataChangeVersion();
List<String> missingJars = getMissingJars(this.remoteFunctionRegistry, localFunctionRegistry, remoteVersion);
List<JarScan> jars = Lists.newArrayList();
if (!missingJars.isEmpty()) {
logger.info("Starting dynamic UDFs lazy-init process.\n" + "The following jars are going to be downloaded and registered locally: " + missingJars);
for (String jarName : missingJars) {
Path binary = null;
Path source = null;
URLClassLoader classLoader = null;
try {
binary = copyJarToLocal(jarName, this.remoteFunctionRegistry);
source = copyJarToLocal(JarUtil.getSourceName(jarName), this.remoteFunctionRegistry);
URL[] urls = { binary.toUri().toURL(), source.toUri().toURL() };
classLoader = new URLClassLoader(urls);
ScanResult scanResult = scan(classLoader, binary, urls);
localFunctionRegistry.validate(jarName, scanResult);
jars.add(new JarScan(jarName, scanResult, classLoader));
} catch (Exception e) {
deleteQuietlyLocalJar(binary);
deleteQuietlyLocalJar(source);
if (classLoader != null) {
try {
classLoader.close();
} catch (Exception ex) {
logger.warn("Problem during closing class loader for {}", jarName, e);
}
}
logger.error("Problem during remote functions load from {}", jarName, e);
}
}
}
long latestRegistryVersion = jars.size() != missingJars.size() ? localRegistryVersion : remoteVersion.getVersion();
localFunctionRegistry.register(jars, latestRegistryVersion);
return true;
}
}
}
return version != localFunctionRegistry.getVersion();
}
use of org.apache.drill.exec.expr.fn.registry.JarScan in project drill by apache.
the class FunctionImplementationRegistry method syncWithRemoteRegistry.
/**
* Purpose of this method is to synchronize remote and local function registries if needed
* and to inform if function registry was changed after given version.
* <p/>
* To make synchronization as much light-weigh as possible, first only versions of both registries are checked
* without any locking. If synchronization is needed, enters synchronized block to prevent others loading the same jars.
* The need of synchronization is checked again (double-check lock) before comparing jars.
* If any missing jars are found, they are downloaded to local udf area, each is wrapped into {@link JarScan}.
* Once jar download is finished, all missing jars are registered in one batch.
* In case if any errors during jars download / registration, these errors are logged.
* <p/>
* During registration local function registry is updated with remote function registry version it is synced with.
* When at least one jar of the missing jars failed to download / register,
* local function registry version are not updated but jars that where successfully downloaded / registered
* are added to local function registry.
* <p/>
* If synchronization between remote and local function registry was not needed,
* checks if given registry version matches latest sync version
* to inform if function registry was changed after given version.
*
* @param version remote function registry local function registry was based on
* @return true if remote and local function registries were synchronized after given version
*/
public boolean syncWithRemoteRegistry(int version) {
// not exist for some JMockit-based unit tests.
if (isRegistrySyncNeeded()) {
synchronized (this) {
int localRegistryVersion = localFunctionRegistry.getVersion();
if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localRegistryVersion)) {
DataChangeVersion remoteVersion = new DataChangeVersion();
List<String> missingJars = getMissingJars(this.remoteFunctionRegistry, localFunctionRegistry, remoteVersion);
List<JarScan> jars = new ArrayList<>();
if (!missingJars.isEmpty()) {
logger.info("Starting dynamic UDFs lazy-init process.\n" + "The following jars are going to be downloaded and registered locally: " + missingJars);
for (String jarName : missingJars) {
Path binary = null;
Path source = null;
URLClassLoader classLoader = null;
try {
binary = copyJarToLocal(jarName, this.remoteFunctionRegistry);
source = copyJarToLocal(JarUtil.getSourceName(jarName), this.remoteFunctionRegistry);
URL[] urls = { binary.toUri().toURL(), source.toUri().toURL() };
classLoader = new URLClassLoader(urls);
ScanResult scanResult = scan(classLoader, binary, urls);
localFunctionRegistry.validate(jarName, scanResult);
jars.add(new JarScan(jarName, scanResult, classLoader));
} catch (Exception e) {
deleteQuietlyLocalJar(binary);
deleteQuietlyLocalJar(source);
if (classLoader != null) {
try {
classLoader.close();
} catch (Exception ex) {
logger.warn("Problem during closing class loader for {}", jarName, e);
}
}
logger.error("Problem during remote functions load from {}", jarName, e);
}
}
}
int latestRegistryVersion = jars.size() != missingJars.size() ? localRegistryVersion : remoteVersion.getVersion();
localFunctionRegistry.register(jars, latestRegistryVersion);
return true;
}
}
}
return version != localFunctionRegistry.getVersion();
}
Aggregations