Examples with DataChangeVersion org.apache.drill.exec.store.sys.store.DataChangeVersion used on opensource projects

Example 6 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by apache.

the class TestDynamicUDFSupport method testDropFunction.

@Test
public void testDropFunction() throws Exception {
    copyDefaultJarsToStagingArea();
    test("create function using jar '%s'", default_binary_name);
    test("select custom_lower('A') from (values(1))");
    Path localUdfDirPath = Deencapsulation.getField(getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir");
    File localUdfDir = new File(localUdfDirPath.toUri().getPath());
    assertTrue("Binary should exist in local udf directory", new File(localUdfDir, default_binary_name).exists());
    assertTrue("Source should exist in local udf directory", new File(localUdfDir, default_source_name).exists());
    String summary = "The following UDFs in jar %s have been unregistered:\n" + "[custom_lower(VARCHAR-REQUIRED)]";
    testBuilder().sqlQuery("drop function using jar '%s'", default_binary_name).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format(summary, default_binary_name)).go();
    try {
        test("select custom_lower('A') from (values(1))");
    } catch (UserRemoteException e) {
        assertThat(e.getMessage(), containsString("No match found for function signature custom_lower(<CHARACTER>)"));
    }
    RemoteFunctionRegistry remoteFunctionRegistry = getDrillbitContext().getRemoteFunctionRegistry();
    assertEquals("Remote registry should be empty", remoteFunctionRegistry.getRegistry(new DataChangeVersion()).getJarList().size(), 0);
    FileSystem fs = remoteFunctionRegistry.getFs();
    assertFalse("Binary should not be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
    assertFalse("Source should not be present in registry area", fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
    assertFalse("Binary should not be present in local udf directory", new File(localUdfDir, default_binary_name).exists());
    assertFalse("Source should not be present in local udf directory", new File(localUdfDir, default_source_name).exists());
}

Example 7 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by apache.

the class TestDynamicUDFSupport method testConcurrentRemoteRegistryUpdateWithDuplicates.

@Test
public void testConcurrentRemoteRegistryUpdateWithDuplicates() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    final CountDownLatch latch1 = new CountDownLatch(1);
    final CountDownLatch latch2 = new CountDownLatch(1);
    final CountDownLatch latch3 = new CountDownLatch(1);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            latch3.countDown();
            latch1.await();
            invocation.callRealMethod();
            latch2.countDown();
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            latch1.countDown();
            latch2.await();
            invocation.callRealMethod();
            return null;
        }
    }).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    final String jarName1 = default_binary_name;
    final String jarName2 = "DrillUDF_Copy-1.0.jar";
    final String query = "create function using jar '%s'";
    copyDefaultJarsToStagingArea();
    copyJarsToStagingArea(jarName2, JarUtil.getSourceName(jarName2));
    Thread thread1 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jarName1).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("The following UDFs in jar %s have been registered:\n" + "[custom_lower(VARCHAR-REQUIRED)]", jarName1))));
    Thread thread2 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jarName2).unOrdered().baselineColumns("ok", "summary").baselineValues(false, String.format("Found duplicated function in %s: custom_lower(VARCHAR-REQUIRED)", jarName1))));
    thread1.start();
    latch3.await();
    thread2.start();
    thread1.join();
    thread2.join();
    DataChangeVersion version = new DataChangeVersion();
    Registry registry = remoteFunctionRegistry.getRegistry(version);
    assertEquals("Remote registry version should match", 1, version.getVersion());
    List<Jar> jarList = registry.getJarList();
    assertEquals("Only one jar should be registered", 1, jarList.size());
    assertEquals("Jar name should match", jarName1, jarList.get(0).getName());
    verify(remoteFunctionRegistry, times(2)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
}

Example 8 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by apache.

the class TestDynamicUDFSupport method testConcurrentRemoteRegistryUpdateForDifferentJars.

@Test
public void testConcurrentRemoteRegistryUpdateForDifferentJars() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    final CountDownLatch latch1 = new CountDownLatch(1);
    final CountDownLatch latch2 = new CountDownLatch(2);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            latch2.countDown();
            latch1.await();
            invocation.callRealMethod();
            return null;
        }
    }).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    final String jarName1 = default_binary_name;
    final String jarName2 = "DrillUDF-2.0.jar";
    final String query = "create function using jar '%s'";
    copyDefaultJarsToStagingArea();
    copyJarsToStagingArea(jarName2, JarUtil.getSourceName(jarName2));
    Thread thread1 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jarName1).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("The following UDFs in jar %s have been registered:\n" + "[custom_lower(VARCHAR-REQUIRED)]", jarName1))));
    Thread thread2 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jarName2).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("The following UDFs in jar %s have been registered:\n" + "[custom_upper(VARCHAR-REQUIRED)]", jarName2))));
    thread1.start();
    thread2.start();
    latch2.await();
    latch1.countDown();
    thread1.join();
    thread2.join();
    DataChangeVersion version = new DataChangeVersion();
    Registry registry = remoteFunctionRegistry.getRegistry(version);
    assertEquals("Remote registry version should match", 2, version.getVersion());
    List<Jar> actualJars = registry.getJarList();
    List<String> expectedJars = Lists.newArrayList(jarName1, jarName2);
    assertEquals("Only one jar should be registered", 2, actualJars.size());
    for (Jar jar : actualJars) {
        assertTrue("Jar should be present in remote function registry", expectedJars.contains(jar.getName()));
    }
    verify(remoteFunctionRegistry, times(3)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
}

Example 9 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by axbaretto.

the class FunctionImplementationRegistry method syncWithRemoteRegistry.

/**
 * Purpose of this method is to synchronize remote and local function registries if needed
 * and to inform if function registry was changed after given version.
 *
 * To make synchronization as much light-weigh as possible, first only versions of both registries are checked
 * without any locking. If synchronization is needed, enters synchronized block to prevent others loading the same jars.
 * The need of synchronization is checked again (double-check lock) before comparing jars.
 * If any missing jars are found, they are downloaded to local udf area, each is wrapped into {@link JarScan}.
 * Once jar download is finished, all missing jars are registered in one batch.
 * In case if any errors during jars download / registration, these errors are logged.
 *
 * During registration local function registry is updated with remote function registry version it is synced with.
 * When at least one jar of the missing jars failed to download / register,
 * local function registry version are not updated but jars that where successfully downloaded / registered
 * are added to local function registry.
 *
 * If synchronization between remote and local function registry was not needed,
 * checks if given registry version matches latest sync version
 * to inform if function registry was changed after given version.
 *
 * @param version remote function registry local function registry was based on
 * @return true if remote and local function registries were synchronized after given version
 */
@SuppressWarnings("resource")
public boolean syncWithRemoteRegistry(long version) {
    // not exist for some JMockit-based unit tests.
    if (isRegistrySyncNeeded()) {
        synchronized (this) {
            long localRegistryVersion = localFunctionRegistry.getVersion();
            if (isRegistrySyncNeeded(remoteFunctionRegistry.getRegistryVersion(), localRegistryVersion)) {
                DataChangeVersion remoteVersion = new DataChangeVersion();
                List<String> missingJars = getMissingJars(this.remoteFunctionRegistry, localFunctionRegistry, remoteVersion);
                List<JarScan> jars = Lists.newArrayList();
                if (!missingJars.isEmpty()) {
                    logger.info("Starting dynamic UDFs lazy-init process.\n" + "The following jars are going to be downloaded and registered locally: " + missingJars);
                    for (String jarName : missingJars) {
                        Path binary = null;
                        Path source = null;
                        URLClassLoader classLoader = null;
                        try {
                            binary = copyJarToLocal(jarName, this.remoteFunctionRegistry);
                            source = copyJarToLocal(JarUtil.getSourceName(jarName), this.remoteFunctionRegistry);
                            URL[] urls = { binary.toUri().toURL(), source.toUri().toURL() };
                            classLoader = new URLClassLoader(urls);
                            ScanResult scanResult = scan(classLoader, binary, urls);
                            localFunctionRegistry.validate(jarName, scanResult);
                            jars.add(new JarScan(jarName, scanResult, classLoader));
                        } catch (Exception e) {
                            deleteQuietlyLocalJar(binary);
                            deleteQuietlyLocalJar(source);
                            if (classLoader != null) {
                                try {
                                    classLoader.close();
                                } catch (Exception ex) {
                                    logger.warn("Problem during closing class loader for {}", jarName, e);
                                }
                            }
                            logger.error("Problem during remote functions load from {}", jarName, e);
                        }
                    }
                }
                long latestRegistryVersion = jars.size() != missingJars.size() ? localRegistryVersion : remoteVersion.getVersion();
                localFunctionRegistry.register(jars, latestRegistryVersion);
                return true;
            }
        }
    }
    return version != localFunctionRegistry.getVersion();
}

Example 10 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by axbaretto.

the class RemoteFunctionRegistry method getRegistryVersion.

/**
 * Returns current remote function registry version.
 * If remote function registry is not found or unreachable, logs error and returns -1.
 *
 * @return remote function registry version if any, -1 otherwise
 */
public long getRegistryVersion() {
    DataChangeVersion version = new DataChangeVersion();
    boolean contains = false;
    try {
        contains = registry.contains(registry_path, version);
    } catch (Exception e) {
        logger.error("Problem during trying to access remote function registry [{}]", registry_path, e);
    }
    if (contains) {
        return version.getVersion();
    } else {
        logger.error("Remote function registry [{}] is unreachable", registry_path);
        return -1;
    }
}