Examples with Jar org.apache.drill.exec.proto.UserBitShared.Jar used on opensource projects

Example 11 with Jar

use of org.apache.drill.exec.proto.UserBitShared.Jar in project drill by apache.

the class CreateFunctionHandler method initRemoteRegistration.

/**
 * Instantiates remote registration. First gets remote function registry with version.
 * Version is used to ensure that we update the same registry we validated against.
 * Then validates against list of remote jars.
 * If validation is successful, first copies jars to registry area and starts updating remote function registry.
 * If during update {@link VersionMismatchException} was detected,
 * attempts to repeat remote registration process till retry attempts exceeds the limit.
 * If retry attempts number hits 0, throws exception that failed to update remote function registry.
 * In case of any error, if jars have been already copied to registry area, they will be deleted.
 *
 * @param functions list of functions present in jar
 * @param jarManager helper class for copying jars to registry area
 * @param remoteRegistry remote function registry
 * @throws IOException in case of problems with copying jars to registry area
 */
private void initRemoteRegistration(List<String> functions, JarManager jarManager, RemoteFunctionRegistry remoteRegistry) throws IOException {
    int retryAttempts = remoteRegistry.getRetryAttempts();
    boolean copyJars = true;
    try {
        while (retryAttempts >= 0) {
            DataChangeVersion version = new DataChangeVersion();
            List<Jar> remoteJars = remoteRegistry.getRegistry(version).getJarList();
            validateAgainstRemoteRegistry(remoteJars, jarManager.getBinaryName(), functions);
            if (copyJars) {
                jarManager.copyToRegistryArea();
                copyJars = false;
            }
            List<Jar> jars = Lists.newArrayList(remoteJars);
            jars.add(Jar.newBuilder().setName(jarManager.getBinaryName()).addAllFunctionSignature(functions).build());
            Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
            try {
                remoteRegistry.updateRegistry(updatedRegistry, version);
                return;
            } catch (VersionMismatchException ex) {
                logger.debug("Failed to update function registry during registration, version mismatch was detected.", ex);
                retryAttempts--;
            }
        }
        throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
    } catch (Exception e) {
        if (!copyJars) {
            jarManager.deleteQuietlyFromRegistryArea();
        }
        throw e;
    }
}

Example 12 with Jar

use of org.apache.drill.exec.proto.UserBitShared.Jar in project drill by apache.

the class DropFunctionHandler method unregister.

/**
 * Gets remote function registry with version.
 * Version is used to ensure that we update the same registry we removed jars from.
 * Looks for a jar to be deleted, if founds one,
 * attempts to update remote registry with list of jars, that excludes jar to be deleted.
 * If during update {@link VersionMismatchException} was detected,
 * attempts to repeat unregistration process till retry attempts exceeds the limit.
 * If retry attempts number hits 0, throws exception that failed to update remote function registry.
 *
 * @param jarName jar name
 * @param remoteFunctionRegistry remote function registry
 * @return jar that was unregistered, null otherwise
 */
private Jar unregister(String jarName, RemoteFunctionRegistry remoteFunctionRegistry) {
    int retryAttempts = remoteFunctionRegistry.getRetryAttempts();
    while (retryAttempts >= 0) {
        DataChangeVersion version = new DataChangeVersion();
        Registry registry = remoteFunctionRegistry.getRegistry(version);
        Jar jarToBeDeleted = null;
        List<Jar> jars = Lists.newArrayList();
        for (Jar j : registry.getJarList()) {
            if (j.getName().equals(jarName)) {
                jarToBeDeleted = j;
            } else {
                jars.add(j);
            }
        }
        if (jarToBeDeleted == null) {
            return null;
        }
        Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
        try {
            remoteFunctionRegistry.updateRegistry(updatedRegistry, version);
            return jarToBeDeleted;
        } catch (VersionMismatchException ex) {
            logger.debug("Failed to update function registry during unregistration, version mismatch was detected.", ex);
            retryAttempts--;
        }
    }
    throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
}

Example 13 with Jar

use of org.apache.drill.exec.proto.UserBitShared.Jar in project drill by axbaretto.

the class DropFunctionHandler method getPlan.

/**
 * Unregisters UDFs dynamically. Process consists of several steps:
 * <ol>
 * <li>Registering jar in jar registry to ensure that several jars with the same name is not being unregistered.</li>
 * <li>Starts remote unregistration process, gets list of all jars and excludes jar to be deleted.</li>
 * <li>Signals drill bits to start local unregistration process.</li>
 * <li>Removes source and binary jars from registry area.</li>
 * </ol>
 *
 * UDFs unregistration is allowed only if dynamic UDFs support is enabled.
 * Only jars registered dynamically can be unregistered,
 * built-in functions loaded at start up are not allowed to be unregistered.
 *
 * Limitation: before jar unregistration make sure no one is using functions from this jar.
 * There is no guarantee that running queries will finish successfully or give correct result.
 *
 * @return - Single row indicating list of unregistered UDFs, raise exception otherwise
 */
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException, IOException {
    if (!context.getOption(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED).bool_val) {
        throw UserException.validationError().message("Dynamic UDFs support is disabled.").build(logger);
    }
    SqlDropFunction node = unwrap(sqlNode, SqlDropFunction.class);
    String jarName = ((SqlCharStringLiteral) node.getJar()).toValue();
    RemoteFunctionRegistry remoteFunctionRegistry = context.getRemoteFunctionRegistry();
    boolean inProgress = false;
    try {
        final String action = remoteFunctionRegistry.addToJars(jarName, RemoteFunctionRegistry.Action.UNREGISTRATION);
        if (!(inProgress = action == null)) {
            return DirectPlan.createDirectPlan(context, false, String.format("Jar with %s name is used. Action: %s", jarName, action));
        }
        Jar deletedJar = unregister(jarName, remoteFunctionRegistry);
        if (deletedJar == null) {
            return DirectPlan.createDirectPlan(context, false, String.format("Jar %s is not registered in remote registry", jarName));
        }
        remoteFunctionRegistry.submitForUnregistration(jarName);
        removeJarFromArea(jarName, remoteFunctionRegistry.getFs(), remoteFunctionRegistry.getRegistryArea());
        removeJarFromArea(JarUtil.getSourceName(jarName), remoteFunctionRegistry.getFs(), remoteFunctionRegistry.getRegistryArea());
        return DirectPlan.createDirectPlan(context, true, String.format("The following UDFs in jar %s have been unregistered:\n%s", jarName, deletedJar.getFunctionSignatureList()));
    } catch (Exception e) {
        logger.error("Error during UDF unregistration", e);
        return DirectPlan.createDirectPlan(context, false, e.getMessage());
    } finally {
        if (inProgress) {
            remoteFunctionRegistry.finishUnregistration(jarName);
            remoteFunctionRegistry.removeFromJars(jarName);
        }
    }
}

Example 14 with Jar

use of org.apache.drill.exec.proto.UserBitShared.Jar in project drill by apache.

the class TestDynamicUDFSupport method testLazyInitConcurrent.

@Test
public void testLazyInitConcurrent() throws Exception {
    FunctionImplementationRegistry functionImplementationRegistry = spyFunctionImplementationRegistry();
    copyDefaultJarsToStagingArea();
    test("create function using jar '%s'", defaultBinaryJar);
    final CountDownLatch latch1 = new CountDownLatch(1);
    final CountDownLatch latch2 = new CountDownLatch(1);
    final String query = "select custom_lower('A') from (values(1))";
    doAnswer(invocation -> {
        latch1.await();
        boolean result = (boolean) invocation.callRealMethod();
        assertTrue("syncWithRemoteRegistry() should return true", result);
        latch2.countDown();
        return true;
    }).doAnswer(invocation -> {
        latch1.countDown();
        latch2.await();
        boolean result = (boolean) invocation.callRealMethod();
        assertTrue("syncWithRemoteRegistry() should return true", result);
        return true;
    }).when(functionImplementationRegistry).syncWithRemoteRegistry(anyInt());
    SimpleQueryRunner simpleQueryRunner = new SimpleQueryRunner(query);
    Thread thread1 = new Thread(simpleQueryRunner);
    Thread thread2 = new Thread(simpleQueryRunner);
    thread1.start();
    thread2.start();
    thread1.join();
    thread2.join();
    verify(functionImplementationRegistry, times(2)).syncWithRemoteRegistry(anyInt());
    LocalFunctionRegistry localFunctionRegistry = (LocalFunctionRegistry) FieldUtils.readField(functionImplementationRegistry, "localFunctionRegistry", true);
    assertEquals("Sync function registry version should match", 2, localFunctionRegistry.getVersion());
}

Example 15 with Jar

use of org.apache.drill.exec.proto.UserBitShared.Jar in project drill by apache.

the class FunctionImplementationRegistry method getMissingJars.

/**
 * Return list of jars that are missing in local function registry
 * but present in remote function registry.
 * Also updates version holder with remote function registry version.
 *
 * @param remoteFunctionRegistry remote function registry
 * @param localFunctionRegistry local function registry
 * @param version holder for remote function registry version
 * @return list of missing jars
 */
private List<String> getMissingJars(RemoteFunctionRegistry remoteFunctionRegistry, LocalFunctionRegistry localFunctionRegistry, DataChangeVersion version) {
    List<Jar> remoteJars = remoteFunctionRegistry.getRegistry(version).getJarList();
    List<String> localJars = localFunctionRegistry.getAllJarNames();
    List<String> missingJars = new ArrayList<>();
    for (Jar jar : remoteJars) {
        if (!localJars.contains(jar.getName())) {
            missingJars.add(jar.getName());
        }
    }
    return missingJars;
}