Examples with RemoteFunctionRegistry org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry used on opensource projects

Example 1 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method testExceedRetryAttemptsDuringUnregistration.

@Test
public void testExceedRetryAttemptsDuringUnregistration() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    copyDefaultJarsToStagingArea();
    test("create function using jar '%s'", default_binary_name);
    reset(remoteFunctionRegistry);
    doThrow(new VersionMismatchException("Version mismatch detected", 1)).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    String summary = "Failed to update remote function registry. Exceeded retry attempts limit.";
    testBuilder().sqlQuery("drop function using jar '%s'", default_binary_name).unOrdered().baselineColumns("ok", "summary").baselineValues(false, summary).go();
    verify(remoteFunctionRegistry, times(remoteFunctionRegistry.getRetryAttempts() + 1)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    FileSystem fs = remoteFunctionRegistry.getFs();
    assertTrue("Binary should be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
    assertTrue("Source should be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
    Registry registry = remoteFunctionRegistry.getRegistry(new DataChangeVersion());
    assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
    assertEquals(registry.getJar(0).getName(), default_binary_name);
}

Example 2 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class CreateFunctionHandler method initRemoteRegistration.

/**
   * Instantiates remote registration. First gets remote function registry with version.
   * Version is used to ensure that we update the same registry we validated against.
   * Then validates against list of remote jars.
   * If validation is successful, first copies jars to registry area and starts updating remote function registry.
   * If during update {@link VersionMismatchException} was detected,
   * attempts to repeat remote registration process till retry attempts exceeds the limit.
   * If retry attempts number hits 0, throws exception that failed to update remote function registry.
   * In case of any error, if jars have been already copied to registry area, they will be deleted.
   *
   * @param functions list of functions present in jar
   * @param jarManager helper class for copying jars to registry area
   * @param remoteRegistry remote function registry
   * @throws IOException in case of problems with copying jars to registry area
   */
private void initRemoteRegistration(List<String> functions, JarManager jarManager, RemoteFunctionRegistry remoteRegistry) throws IOException {
    int retryAttempts = remoteRegistry.getRetryAttempts();
    boolean copyJars = true;
    try {
        while (retryAttempts >= 0) {
            DataChangeVersion version = new DataChangeVersion();
            List<Jar> remoteJars = remoteRegistry.getRegistry(version).getJarList();
            validateAgainstRemoteRegistry(remoteJars, jarManager.getBinaryName(), functions);
            if (copyJars) {
                jarManager.copyToRegistryArea();
                copyJars = false;
            }
            List<Jar> jars = Lists.newArrayList(remoteJars);
            jars.add(Jar.newBuilder().setName(jarManager.getBinaryName()).addAllFunctionSignature(functions).build());
            Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
            try {
                remoteRegistry.updateRegistry(updatedRegistry, version);
                return;
            } catch (VersionMismatchException ex) {
                logger.debug("Failed to update function registry during registration, version mismatch was detected.", ex);
                retryAttempts--;
            }
        }
        throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
    } catch (Exception e) {
        if (!copyJars) {
            jarManager.deleteQuietlyFromRegistryArea();
        }
        throw e;
    }
}

Example 3 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class DropFunctionHandler method unregister.

/**
   * Gets remote function registry with version.
   * Version is used to ensure that we update the same registry we removed jars from.
   * Looks for a jar to be deleted, if founds one,
   * attempts to update remote registry with list of jars, that excludes jar to be deleted.
   * If during update {@link VersionMismatchException} was detected,
   * attempts to repeat unregistration process till retry attempts exceeds the limit.
   * If retry attempts number hits 0, throws exception that failed to update remote function registry.
   *
   * @param jarName jar name
   * @param remoteFunctionRegistry remote function registry
   * @return jar that was unregistered, null otherwise
   */
private Jar unregister(String jarName, RemoteFunctionRegistry remoteFunctionRegistry) {
    int retryAttempts = remoteFunctionRegistry.getRetryAttempts();
    while (retryAttempts >= 0) {
        DataChangeVersion version = new DataChangeVersion();
        Registry registry = remoteFunctionRegistry.getRegistry(version);
        Jar jarToBeDeleted = null;
        List<Jar> jars = Lists.newArrayList();
        for (Jar j : registry.getJarList()) {
            if (j.getName().equals(jarName)) {
                jarToBeDeleted = j;
            } else {
                jars.add(j);
            }
        }
        if (jarToBeDeleted == null) {
            return null;
        }
        Registry updatedRegistry = Registry.newBuilder().addAllJar(jars).build();
        try {
            remoteFunctionRegistry.updateRegistry(updatedRegistry, version);
            return jarToBeDeleted;
        } catch (VersionMismatchException ex) {
            logger.debug("Failed to update function registry during unregistration, version mismatch was detected.", ex);
            retryAttempts--;
        }
    }
    throw new DrillRuntimeException("Failed to update remote function registry. Exceeded retry attempts limit.");
}

Example 4 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method testConcurrentRegistrationOfTheSameJar.

@Test
public void testConcurrentRegistrationOfTheSameJar() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    final CountDownLatch latch1 = new CountDownLatch(1);
    final CountDownLatch latch2 = new CountDownLatch(1);
    doAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            String result = (String) invocation.callRealMethod();
            latch2.countDown();
            latch1.await();
            return result;
        }
    }).doCallRealMethod().doCallRealMethod().when(remoteFunctionRegistry).addToJars(anyString(), any(RemoteFunctionRegistry.Action.class));
    final String query = String.format("create function using jar '%s'", default_binary_name);
    Thread thread = new Thread(new SimpleQueryRunner(query));
    thread.start();
    latch2.await();
    try {
        String summary = "Jar with %s name is used. Action: REGISTRATION";
        testBuilder().sqlQuery(query).unOrdered().baselineColumns("ok", "summary").baselineValues(false, String.format(summary, default_binary_name)).go();
        testBuilder().sqlQuery("drop function using jar '%s'", default_binary_name).unOrdered().baselineColumns("ok", "summary").baselineValues(false, String.format(summary, default_binary_name)).go();
    } finally {
        latch1.countDown();
        thread.join();
    }
}

Example 5 with RemoteFunctionRegistry

use of org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry in project drill by apache.

the class TestDynamicUDFSupport method testExceedRetryAttemptsDuringRegistration.

@Test
public void testExceedRetryAttemptsDuringRegistration() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    copyDefaultJarsToStagingArea();
    doThrow(new VersionMismatchException("Version mismatch detected", 1)).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    String summary = "Failed to update remote function registry. Exceeded retry attempts limit.";
    testBuilder().sqlQuery("create function using jar '%s'", default_binary_name).unOrdered().baselineColumns("ok", "summary").baselineValues(false, summary).go();
    verify(remoteFunctionRegistry, times(remoteFunctionRegistry.getRetryAttempts() + 1)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    FileSystem fs = remoteFunctionRegistry.getFs();
    assertTrue("Binary should be present in staging area", fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_binary_name)));
    assertTrue("Source should be present in staging area", fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_source_name)));
    assertFalse("Registry area should be empty", fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
    assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
    assertEquals("Registry should be empty", remoteFunctionRegistry.getRegistry(new DataChangeVersion()).getJarList().size(), 0);
}