Examples with ScanResult org.apache.drill.common.scanner.persistence.ScanResult used on opensource projects

Example 6 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class FunctionImplementationRegistry method syncWithRemoteRegistry.

/**
 * Purpose of this method is to synchronize remote and local function registries if needed
 * and to inform if function registry was changed after given version.
 *
 * To make synchronization as much light-weigh as possible, first only versions of both registries are checked
 * without any locking. If synchronization is needed, enters synchronized block to prevent others loading the same jars.
 * The need of synchronization is checked again (double-check lock) before comparing jars.
 * If any missing jars are found, they are downloaded to local udf area, each is wrapped into {@link JarScan}.
 * Once jar download is finished, all missing jars are registered in one batch.
 * In case if any errors during jars download / registration, these errors are logged.
 *
 * During registration local function registry is updated with remote function registry version it is synced with.
 * When at least one jar of the missing jars failed to download / register,
 * local function registry version are not updated but jars that where successfully downloaded / registered
 * are added to local function registry.
 *
 * If synchronization between remote and local function registry was not needed,
 * checks if given registry version matches latest sync version
 * to inform if function registry was changed after given version.
 *
 * @param version remote function registry local function registry was based on
 * @return true if remote and local function registries were synchronized after given version
 */
@SuppressWarnings("resource")
public boolean syncWithRemoteRegistry(long version) {
    // not exist for some JMockit-based unit tests.
    if (isRegistrySyncNeeded()) {
        synchronized (this) {
            long localRegistryVersion = localFunctionRegistry.getVersion();
            if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localRegistryVersion)) {
                DataChangeVersion remoteVersion = new DataChangeVersion();
                List<String> missingJars = getMissingJars(this.remoteFunctionRegistry, localFunctionRegistry, remoteVersion);
                List<JarScan> jars = Lists.newArrayList();
                if (!missingJars.isEmpty()) {
                    logger.info("Starting dynamic UDFs lazy-init process.\n" + "The following jars are going to be downloaded and registered locally: " + missingJars);
                    for (String jarName : missingJars) {
                        Path binary = null;
                        Path source = null;
                        URLClassLoader classLoader = null;
                        try {
                            binary = copyJarToLocal(jarName, this.remoteFunctionRegistry);
                            source = copyJarToLocal(JarUtil.getSourceName(jarName), this.remoteFunctionRegistry);
                            URL[] urls = { binary.toUri().toURL(), source.toUri().toURL() };
                            classLoader = new URLClassLoader(urls);
                            ScanResult scanResult = scan(classLoader, binary, urls);
                            localFunctionRegistry.validate(jarName, scanResult);
                            jars.add(new JarScan(jarName, scanResult, classLoader));
                        } catch (Exception e) {
                            deleteQuietlyLocalJar(binary);
                            deleteQuietlyLocalJar(source);
                            if (classLoader != null) {
                                try {
                                    classLoader.close();
                                } catch (Exception ex) {
                                    logger.warn("Problem during closing class loader for {}", jarName, e);
                                }
                            }
                            logger.error("Problem during remote functions load from {}", jarName, e);
                        }
                    }
                }
                long latestRegistryVersion = jars.size() != missingJars.size() ? localRegistryVersion : remoteVersion.getVersion();
                localFunctionRegistry.register(jars, latestRegistryVersion);
                return true;
            }
        }
    }
    return version != localFunctionRegistry.getVersion();
}

Example 7 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class FunctionImplementationRegistry method validate.

/**
 * Using given local path to jar creates unique class loader for this jar.
 * Class loader is closed to release opened connection to jar when validation is finished.
 * Scan jar content to receive list of all scanned classes
 * and starts validation process against local function registry.
 * Checks if received list of validated function is not empty.
 *
 * @param path local path to jar we need to validate
 * @return list of validated function signatures
 */
public List<String> validate(Path path) throws IOException {
    URL url = path.toUri().toURL();
    URL[] urls = { url };
    try (URLClassLoader classLoader = new URLClassLoader(urls)) {
        ScanResult jarScanResult = scan(classLoader, path, urls);
        List<String> functions = localFunctionRegistry.validate(path.getName(), jarScanResult);
        if (functions.isEmpty()) {
            throw new FunctionValidationException(String.format("Jar %s does not contain functions", path.getName()));
        }
        return functions;
    }
}

Example 8 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class Drillbit method start.

@VisibleForTesting
public static Drillbit start(final DrillConfig config, final CaseInsensitiveMap<OptionDefinition> validators, final RemoteServiceSet remoteServiceSet) throws DrillbitStartupException {
    logger.debug("Starting new Drillbit.");
    // TODO: allow passing as a parameter
    ScanResult classpathScan = ClassPathScanner.fromPrescan(config);
    Drillbit bit;
    try {
        bit = new Drillbit(config, validators, remoteServiceSet, classpathScan);
    } catch (final Exception ex) {
        throw new DrillbitStartupException("Failure while initializing values in Drillbit.", ex);
    }
    try {
        bit.run();
    } catch (final Exception e) {
        logger.error("Failure during initial startup of Drillbit.", e);
        bit.close();
        throw new DrillbitStartupException("Failure during initial startup of Drillbit.", e);
    }
    logger.debug("Started new Drillbit.");
    return bit;
}

Example 9 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class TestClassPathScanner method test.

@Test
public void test() throws Exception {
    ScanResult result = ClassPathScanner.fromPrescan(DrillConfig.create());
    List<AnnotatedClassDescriptor> functions = result.getAnnotatedClasses();
    Set<String> scanned = new HashSet<>();
    AnnotatedClassDescriptor functionRandomBigIntGauss = null;
    for (AnnotatedClassDescriptor function : functions) {
        assertTrue(function.getClassName() + " scanned twice", scanned.add(function.getClassName()));
        if (function.getClassName().equals(RandomBigIntGauss.class.getName())) {
            functionRandomBigIntGauss = function;
        }
    }
    if (functionRandomBigIntGauss == null) {
        Assert.fail("functionRandomBigIntGauss not found");
    }
    // TODO: use Andrew's randomized test framework to verify a subset of the functions
    for (AnnotatedClassDescriptor function : functions) {
        Class<?> c = Class.forName(function.getClassName(), false, this.getClass().getClassLoader());
        Field[] fields = c.getDeclaredFields();
        assertEquals("fields count for " + function, fields.length, function.getFields().size());
        for (int i = 0; i < fields.length; i++) {
            FieldDescriptor fieldDescriptor = function.getFields().get(i);
            Field field = fields[i];
            assertEquals("Class fields:\n" + Arrays.toString(fields) + "\n != \nDescriptor fields:\n" + function.getFields(), field.getName(), fieldDescriptor.getName());
            verifyAnnotations(field.getDeclaredAnnotations(), fieldDescriptor.getAnnotations());
            assertEquals(field.getType(), fieldDescriptor.getFieldClass());
        }
        Annotation[] annotations = c.getDeclaredAnnotations();
        List<AnnotationDescriptor> scannedAnnotations = function.getAnnotations();
        verifyAnnotations(annotations, scannedAnnotations);
        FunctionTemplate bytecodeAnnotation = function.getAnnotationProxy(FunctionTemplate.class);
        FunctionTemplate reflectionAnnotation = c.getAnnotation(FunctionTemplate.class);
        assertEquals(reflectionAnnotation.name(), bytecodeAnnotation.name());
        assertArrayEquals(reflectionAnnotation.names(), bytecodeAnnotation.names());
        assertEquals(reflectionAnnotation.scope(), bytecodeAnnotation.scope());
        assertEquals(reflectionAnnotation.nulls(), bytecodeAnnotation.nulls());
        assertEquals(reflectionAnnotation.isBinaryCommutative(), bytecodeAnnotation.isBinaryCommutative());
        assertEquals(reflectionAnnotation.desc(), bytecodeAnnotation.desc());
        assertEquals(reflectionAnnotation.costCategory(), bytecodeAnnotation.costCategory());
    }
    for (String baseType : result.getScannedClasses()) {
        validateType(result, baseType);
    }
    assertTrue(result.getImplementations(PhysicalOperator.class).size() > 0);
    assertTrue(result.getImplementations(DrillFunc.class).size() > 0);
}

Example 10 with ScanResult

use of org.apache.drill.common.scanner.persistence.ScanResult in project drill by axbaretto.

the class TestSpnegoAuthentication method testFORMEnabledWithPlainDisabled.

/**
 * Validate failure in creating FORM security handler when PAM authenticator is absent. PAM authenticator is provided
 * via {@link PlainFactory#getAuthenticator()}
 * @throws Exception
 */
@Test
public void testFORMEnabledWithPlainDisabled() throws Exception {
    try {
        final DrillConfig newConfig = new DrillConfig(DrillConfig.create().withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.HTTP_AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("form"))).withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)).withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())));
        final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig);
        final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class);
        Mockito.when(authenticatorProvider.containsFactory(PlainFactory.SIMPLE_NAME)).thenReturn(false);
        final DrillbitContext context = Mockito.mock(DrillbitContext.class);
        Mockito.when(context.getClasspathScan()).thenReturn(scanResult);
        Mockito.when(context.getConfig()).thenReturn(newConfig);
        Mockito.when(context.getAuthProvider()).thenReturn(authenticatorProvider);
        final DrillHttpSecurityHandlerProvider securityProvider = new DrillHttpSecurityHandlerProvider(newConfig, context);
        fail();
    } catch (Exception ex) {
        assertTrue(ex instanceof DrillbitStartupException);
    }
}