Examples with RemoteFunctionRegistry org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry used on opensource projects

Example 11 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method spyRemoteFunctionRegistry.

private RemoteFunctionRegistry spyRemoteFunctionRegistry() {
    FunctionImplementationRegistry functionImplementationRegistry = getDrillbitContext().getFunctionImplementationRegistry();
    RemoteFunctionRegistry remoteFunctionRegistry = functionImplementationRegistry.getRemoteFunctionRegistry();
    RemoteFunctionRegistry spy = spy(remoteFunctionRegistry);
    Deencapsulation.setField(functionImplementationRegistry, "remoteFunctionRegistry", spy);
    return spy;
}

Example 12 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method testSuccessfulRegistrationAfterSeveralRetryAttempts.

@Test
public void testSuccessfulRegistrationAfterSeveralRetryAttempts() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    copyDefaultJarsToStagingArea();
    doThrow(new VersionMismatchException("Version mismatch detected", 1)).doThrow(new VersionMismatchException("Version mismatch detected", 1)).doCallRealMethod().when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    String summary = "The following UDFs in jar %s have been registered:\n" + "[custom_lower(VARCHAR-REQUIRED)]";
    testBuilder().sqlQuery("create function using jar '%s'", default_binary_name).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format(summary, default_binary_name)).go();
    verify(remoteFunctionRegistry, times(3)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    FileSystem fs = remoteFunctionRegistry.getFs();
    assertFalse("Staging area should be empty", fs.listFiles(remoteFunctionRegistry.getStagingArea(), false).hasNext());
    assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
    assertTrue("Binary should be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
    assertTrue("Source should be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
    Registry registry = remoteFunctionRegistry.getRegistry(new DataChangeVersion());
    assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
    assertEquals(registry.getJar(0).getName(), default_binary_name);
}

Example 13 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method testRegistrationFailDuringRegistryUpdate.

@Test
public void testRegistrationFailDuringRegistryUpdate() throws Exception {
    final RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    final FileSystem fs = remoteFunctionRegistry.getFs();
    final String errorMessage = "Failure during remote registry update.";
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            assertTrue("Binary should be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
            assertTrue("Source should be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
            throw new RuntimeException(errorMessage);
        }
    }).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    copyDefaultJarsToStagingArea();
    testBuilder().sqlQuery("create function using jar '%s'", default_binary_name).unOrdered().baselineColumns("ok", "summary").baselineValues(false, errorMessage).go();
    assertFalse("Registry area should be empty", fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
    assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
    assertTrue("Binary should be present in staging area", fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_binary_name)));
    assertTrue("Source should be present in staging area", fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_source_name)));
}

Example 14 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method copyJarsToStagingArea.

private void copyJarsToStagingArea(Path src, String binaryName, String sourceName) throws IOException {
    RemoteFunctionRegistry remoteFunctionRegistry = getDrillbitContext().getRemoteFunctionRegistry();
    copyJar(remoteFunctionRegistry.getFs(), src, remoteFunctionRegistry.getStagingArea(), binaryName);
    copyJar(remoteFunctionRegistry.getFs(), src, remoteFunctionRegistry.getStagingArea(), sourceName);
}

Example 15 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class DropFunctionHandler method getPlan.

/**
   * Unregisters UDFs dynamically. Process consists of several steps:
   * <ol>
   * <li>Registering jar in jar registry to ensure that several jars with the same name is not being unregistered.</li>
   * <li>Starts remote unregistration process, gets list of all jars and excludes jar to be deleted.</li>
   * <li>Signals drill bits to start local unregistration process.</li>
   * <li>Removes source and binary jars from registry area.</li>
   * </ol>
   *
   * UDFs unregistration is allowed only if dynamic UDFs support is enabled.
   * Only jars registered dynamically can be unregistered,
   * built-in functions loaded at start up are not allowed to be unregistered.
   *
   * Limitation: before jar unregistration make sure no one is using functions from this jar.
   * There is no guarantee that running queries will finish successfully or give correct result.
   *
   * @return - Single row indicating list of unregistered UDFs, raise exception otherwise
   */
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException, IOException {
    if (!context.getOption(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED).bool_val) {
        throw UserException.validationError().message("Dynamic UDFs support is disabled.").build(logger);
    }
    SqlDropFunction node = unwrap(sqlNode, SqlDropFunction.class);
    String jarName = ((SqlCharStringLiteral) node.getJar()).toValue();
    RemoteFunctionRegistry remoteFunctionRegistry = context.getRemoteFunctionRegistry();
    boolean inProgress = false;
    try {
        final String action = remoteFunctionRegistry.addToJars(jarName, RemoteFunctionRegistry.Action.UNREGISTRATION);
        if (!(inProgress = action == null)) {
            return DirectPlan.createDirectPlan(context, false, String.format("Jar with %s name is used. Action: %s", jarName, action));
        }
        Jar deletedJar = unregister(jarName, remoteFunctionRegistry);
        if (deletedJar == null) {
            return DirectPlan.createDirectPlan(context, false, String.format("Jar %s is not registered in remote registry", jarName));
        }
        remoteFunctionRegistry.submitForUnregistration(jarName);
        removeJarFromArea(jarName, remoteFunctionRegistry.getFs(), remoteFunctionRegistry.getRegistryArea());
        removeJarFromArea(JarUtil.getSourceName(jarName), remoteFunctionRegistry.getFs(), remoteFunctionRegistry.getRegistryArea());
        return DirectPlan.createDirectPlan(context, true, String.format("The following UDFs in jar %s have been unregistered:\n%s", jarName, deletedJar.getFunctionSignatureList()));
    } catch (Exception e) {
        logger.error("Error during UDF unregistration", e);
        return DirectPlan.createDirectPlan(context, false, e.getMessage());
    } finally {
        if (inProgress) {
            remoteFunctionRegistry.finishUnregistration(jarName);
            remoteFunctionRegistry.removeFromJars(jarName);
        }
    }
}