Examples with ScanResult org.apache.drill.common.scanner.persistence.ScanResult used on opensource projects

Example 26 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by apache.

the class FunctionImplementationRegistry method syncWithRemoteRegistry.

/**
 * Purpose of this method is to synchronize remote and local function registries if needed
 * and to inform if function registry was changed after given version.
 * <p/>
 * To make synchronization as much light-weigh as possible, first only versions of both registries are checked
 * without any locking. If synchronization is needed, enters synchronized block to prevent others loading the same jars.
 * The need of synchronization is checked again (double-check lock) before comparing jars.
 * If any missing jars are found, they are downloaded to local udf area, each is wrapped into {@link JarScan}.
 * Once jar download is finished, all missing jars are registered in one batch.
 * In case if any errors during jars download / registration, these errors are logged.
 * <p/>
 * During registration local function registry is updated with remote function registry version it is synced with.
 * When at least one jar of the missing jars failed to download / register,
 * local function registry version are not updated but jars that where successfully downloaded / registered
 * are added to local function registry.
 * <p/>
 * If synchronization between remote and local function registry was not needed,
 * checks if given registry version matches latest sync version
 * to inform if function registry was changed after given version.
 *
 * @param version remote function registry local function registry was based on
 * @return true if remote and local function registries were synchronized after given version
 */
public boolean syncWithRemoteRegistry(int version) {
    // not exist for some JMockit-based unit tests.
    if (isRegistrySyncNeeded()) {
        synchronized (this) {
            int localRegistryVersion = localFunctionRegistry.getVersion();
            if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localRegistryVersion)) {
                DataChangeVersion remoteVersion = new DataChangeVersion();
                List<String> missingJars = getMissingJars(this.remoteFunctionRegistry, localFunctionRegistry, remoteVersion);
                List<JarScan> jars = new ArrayList<>();
                if (!missingJars.isEmpty()) {
                    logger.info("Starting dynamic UDFs lazy-init process.\n" + "The following jars are going to be downloaded and registered locally: " + missingJars);
                    for (String jarName : missingJars) {
                        Path binary = null;
                        Path source = null;
                        URLClassLoader classLoader = null;
                        try {
                            binary = copyJarToLocal(jarName, this.remoteFunctionRegistry);
                            source = copyJarToLocal(JarUtil.getSourceName(jarName), this.remoteFunctionRegistry);
                            URL[] urls = { binary.toUri().toURL(), source.toUri().toURL() };
                            classLoader = new URLClassLoader(urls);
                            ScanResult scanResult = scan(classLoader, binary, urls);
                            localFunctionRegistry.validate(jarName, scanResult);
                            jars.add(new JarScan(jarName, scanResult, classLoader));
                        } catch (Exception e) {
                            deleteQuietlyLocalJar(binary);
                            deleteQuietlyLocalJar(source);
                            if (classLoader != null) {
                                try {
                                    classLoader.close();
                                } catch (Exception ex) {
                                    logger.warn("Problem during closing class loader for {}", jarName, e);
                                }
                            }
                            logger.error("Problem during remote functions load from {}", jarName, e);
                        }
                    }
                }
                int latestRegistryVersion = jars.size() != missingJars.size() ? localRegistryVersion : remoteVersion.getVersion();
                localFunctionRegistry.register(jars, latestRegistryVersion);
                return true;
            }
        }
    }
    return version != localFunctionRegistry.getVersion();
}

Example 27 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class StoragePlugins method main.

public static void main(String[] args) throws Exception {
    DrillConfig config = DrillConfig.create();
    ScanResult scanResult = ClassPathScanner.fromPrescan(config);
    LogicalPlanPersistence lpp = new LogicalPlanPersistence(config, scanResult);
    String data = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
    StoragePlugins se = lpp.getMapper().readValue(data, StoragePlugins.class);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    lpp.getMapper().writeValue(System.out, se);
    lpp.getMapper().writeValue(os, se);
    se = lpp.getMapper().readValue(new ByteArrayInputStream(os.toByteArray()), StoragePlugins.class);
    System.out.println(se);
}

Example 28 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class BuildTimeScan method loadExcept.

/**
 * loads all the prescanned resources from classpath
 * (except for the target location in case it already exists)
 * @return the result of the previous scan
 */
private static ScanResult loadExcept(URL ignored) {
    Set<URL> preScanned = ClassPathScanner.forResource(REGISTRY_FILE, false);
    ScanResult result = null;
    for (URL u : preScanned) {
        if (ignored != null && u.toString().startsWith(ignored.toString())) {
            continue;
        }
        try (InputStream reflections = u.openStream()) {
            ScanResult ref = reader.readValue(reflections);
            if (result == null) {
                result = ref;
            } else {
                result = result.merge(ref);
            }
        } catch (IOException e) {
            throw new DrillRuntimeException("can't read function registry at " + u, e);
        }
    }
    if (result != null) {
        if (logger.isInfoEnabled()) {
            StringBuilder sb = new StringBuilder();
            sb.append(format("Loaded prescanned packages %s from locations:\n", result.getScannedPackages()));
            for (URL u : preScanned) {
                sb.append('\t');
                sb.append(u.toExternalForm());
                sb.append('\n');
            }
        }
        logger.info(format("Loaded prescanned packages %s from locations %s", result.getScannedPackages(), preScanned));
        return result;
    } else {
        return ClassPathScanner.emptyResult();
    }
}

Example 29 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class BuildTimeScan method main.

/**
 * to generate the prescan file during build
 * @param args the root path for the classes where {@link BuildTimeScan#REGISTRY_FILE} is generated
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new IllegalArgumentException("Usage: java {cp} " + BuildTimeScan.class.getName() + " path/to/scan");
    }
    String basePath = args[0];
    logger.info("Scanning: {}", basePath);
    File registryFile = new File(basePath, REGISTRY_FILE);
    File dir = registryFile.getParentFile();
    if ((!dir.exists() && !dir.mkdirs()) || !dir.isDirectory()) {
        throw new IllegalArgumentException("could not create dir " + dir.getAbsolutePath());
    }
    DrillConfig config = DrillConfig.create();
    // normalize
    if (!basePath.endsWith("/")) {
        basePath = basePath + "/";
    }
    if (!basePath.startsWith("/")) {
        basePath = "/" + basePath;
    }
    URL url = new URL("file:" + basePath);
    Set<URL> markedPaths = ClassPathScanner.getMarkedPaths();
    if (!markedPaths.contains(url)) {
        throw new IllegalArgumentException(url + " not in " + markedPaths);
    }
    List<String> packagePrefixes = ClassPathScanner.getPackagePrefixes(config);
    List<String> baseClasses = ClassPathScanner.getScannedBaseClasses(config);
    List<String> scannedAnnotations = ClassPathScanner.getScannedAnnotations(config);
    ScanResult preScanned = loadExcept(url);
    ScanResult scan = ClassPathScanner.scan(asList(url), packagePrefixes, baseClasses, scannedAnnotations, preScanned);
    save(scan, registryFile);
}

Example 30 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class ClassPathScanner method scan.

/**
 * @param pathsToScan the locations to scan for .class files
 * @param packagePrefixes the whitelist of package prefixes to scan
 * @param parentResult if there was a prescan, its result
 * @return the merged scan
 */
static ScanResult scan(Collection<URL> pathsToScan, Collection<String> packagePrefixes, Collection<String> scannedClasses, Collection<String> scannedAnnotations, ScanResult parentResult) {
    Stopwatch watch = Stopwatch.createStarted();
    try {
        AnnotationScanner annotationScanner = new AnnotationScanner(scannedAnnotations);
        SubTypesScanner subTypesScanner = new SubTypesScanner(parentResult.getImplementations());
        if (packagePrefixes.size() > 0) {
            final FilterBuilder filter = new FilterBuilder();
            for (String prefix : packagePrefixes) {
                filter.include(FilterBuilder.prefix(prefix));
            }
            ConfigurationBuilder conf = new ConfigurationBuilder().setUrls(pathsToScan).setMetadataAdapter(// Scanners depend on this
            METADATA_ADAPTER).filterInputsBy(filter).setScanners(annotationScanner, subTypesScanner);
            // scans stuff, but don't use the funky storage layer
            new Reflections(conf);
        }
        List<ParentClassDescriptor> implementations = new ArrayList<>();
        for (String baseTypeName : scannedClasses) {
            implementations.add(new ParentClassDescriptor(baseTypeName, new ArrayList<>(subTypesScanner.getChildrenOf(baseTypeName))));
        }
        List<AnnotatedClassDescriptor> annotated = annotationScanner.getAnnotatedClasses();
        verifyClassUnicity(annotated, pathsToScan);
        return new ScanResult(packagePrefixes, scannedClasses, scannedAnnotations, annotated, implementations);
    } finally {
        logger.info(format("Scanning packages %s in locations %s took %dms", packagePrefixes, pathsToScan, watch.elapsed(MILLISECONDS)));
    }
}