Search in sources :

Example 16 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by axbaretto.

the class TestZookeeperClient method testHasPathFalseWithVersion.

@Test
public void testHasPathFalseWithVersion() {
    DataChangeVersion version0 = new DataChangeVersion();
    version0.setVersion(-1);
    assertFalse(client.hasPath("unknown_path", true, version0));
    assertEquals("Versions should not have changed", -1, version0.getVersion());
}
Also used : DataChangeVersion(org.apache.drill.exec.store.sys.store.DataChangeVersion) Test(org.junit.Test)

Example 17 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by axbaretto.

the class TestZookeeperClient method testPutWithMatchingVersion.

@Test
public void testPutWithMatchingVersion() {
    client.put(path, data);
    DataChangeVersion version = new DataChangeVersion();
    client.get(path, version);
    client.put(path, data, version);
}
Also used : DataChangeVersion(org.apache.drill.exec.store.sys.store.DataChangeVersion) Test(org.junit.Test)

Example 18 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by apache.

the class TestDynamicUDFSupport method testConcurrentRemoteRegistryUpdateWithDuplicates.

@Test
public void testConcurrentRemoteRegistryUpdateWithDuplicates() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    final CountDownLatch latch1 = new CountDownLatch(1);
    final CountDownLatch latch2 = new CountDownLatch(1);
    final CountDownLatch latch3 = new CountDownLatch(1);
    doAnswer(invocation -> {
        latch3.countDown();
        latch1.await();
        invocation.callRealMethod();
        latch2.countDown();
        return null;
    }).doAnswer(invocation -> {
        latch1.countDown();
        latch2.await();
        invocation.callRealMethod();
        return null;
    }).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    final String jar1 = defaultBinaryJar;
    copyDefaultJarsToStagingArea();
    final String copyJarName = "drill-custom-lower-copy";
    final String jar2 = buildAndCopyJarsToStagingArea(copyJarName, "**/CustomLowerFunction.java", null);
    final String query = "create function using jar '%s'";
    Thread thread1 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jar1).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("The following UDFs in jar %s have been registered:\n" + "[custom_lower(VARCHAR-REQUIRED)]", jar1))));
    Thread thread2 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jar2).unOrdered().baselineColumns("ok", "summary").baselineValues(false, String.format("Found duplicated function in %s: custom_lower(VARCHAR-REQUIRED)", jar1))));
    thread1.start();
    latch3.await();
    thread2.start();
    thread1.join();
    thread2.join();
    DataChangeVersion version = new DataChangeVersion();
    Registry registry = remoteFunctionRegistry.getRegistry(version);
    assertEquals("Remote registry version should match", 2, version.getVersion());
    List<Jar> jarList = registry.getJarList();
    assertEquals("Only one jar should be registered", 1, jarList.size());
    assertEquals("Jar name should match", jar1, jarList.get(0).getName());
    verify(remoteFunctionRegistry, times(2)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
}
Also used : RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) FileSystem(org.apache.hadoop.fs.FileSystem) LocalFunctionRegistry(org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) Mockito.doThrow(org.mockito.Mockito.doThrow) TestBuilder(org.apache.drill.test.TestBuilder) SlowTest(org.apache.drill.categories.SlowTest) After(org.junit.After) Mockito.doAnswer(org.mockito.Mockito.doAnswer) HadoopUtils.hadoopToJavaPath(org.apache.drill.test.HadoopUtils.hadoopToJavaPath) Assert.fail(org.junit.Assert.fail) URI(java.net.URI) Path(java.nio.file.Path) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) ConfigConstants(org.apache.drill.common.config.ConfigConstants) Category(org.junit.experimental.categories.Category) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) SqlFunctionTest(org.apache.drill.categories.SqlFunctionTest) Assert.assertFalse(org.junit.Assert.assertFalse) VersionMismatchException(org.apache.drill.exec.exception.VersionMismatchException) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) BeforeClass(org.junit.BeforeClass) BaseTestQuery(org.apache.drill.test.BaseTestQuery) ArrayUtils(org.apache.commons.lang3.ArrayUtils) Mockito.spy(org.mockito.Mockito.spy) DrillbitContext(org.apache.drill.exec.server.DrillbitContext) RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) FieldUtils(org.apache.commons.lang3.reflect.FieldUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) ExpectedException(org.junit.rules.ExpectedException) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) Before(org.junit.Before) UserRemoteException(org.apache.drill.common.exceptions.UserRemoteException) Assert.assertTrue(org.junit.Assert.assertTrue) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) IOException(java.io.IOException) Mockito.times(org.mockito.Mockito.times) DataChangeVersion(org.apache.drill.exec.store.sys.store.DataChangeVersion) File(java.io.File) Mockito.verify(org.mockito.Mockito.verify) Registry(org.apache.drill.exec.proto.UserBitShared.Registry) Lists(org.apache.drill.shaded.guava.com.google.common.collect.Lists) Rule(org.junit.Rule) Paths(java.nio.file.Paths) Jar(org.apache.drill.exec.proto.UserBitShared.Jar) JarUtil(org.apache.drill.exec.util.JarUtil) Mockito.reset(org.mockito.Mockito.reset) Assert.assertEquals(org.junit.Assert.assertEquals) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Jar(org.apache.drill.exec.proto.UserBitShared.Jar) LocalFunctionRegistry(org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) Registry(org.apache.drill.exec.proto.UserBitShared.Registry) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) CountDownLatch(java.util.concurrent.CountDownLatch) DataChangeVersion(org.apache.drill.exec.store.sys.store.DataChangeVersion) SlowTest(org.apache.drill.categories.SlowTest) SqlFunctionTest(org.apache.drill.categories.SqlFunctionTest) Test(org.junit.Test)

Example 19 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by apache.

the class TestDynamicUDFSupport method testConcurrentRemoteRegistryUpdateForDifferentJars.

@Test
public void testConcurrentRemoteRegistryUpdateForDifferentJars() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    final CountDownLatch latch1 = new CountDownLatch(1);
    final CountDownLatch latch2 = new CountDownLatch(2);
    doAnswer(invocation -> {
        latch2.countDown();
        latch1.await();
        invocation.callRealMethod();
        return null;
    }).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    final String jar1 = defaultBinaryJar;
    copyDefaultJarsToStagingArea();
    final String upperJarName = "drill-custom-upper";
    final String jar2 = buildAndCopyJarsToStagingArea(upperJarName, "**/CustomUpperFunction.java", null);
    final String query = "create function using jar '%s'";
    Thread thread1 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jar1).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("The following UDFs in jar %s have been registered:\n" + "[custom_lower(VARCHAR-REQUIRED)]", jar1))));
    Thread thread2 = new Thread(new TestBuilderRunner(testBuilder().sqlQuery(query, jar2).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("The following UDFs in jar %s have been registered:\n" + "[custom_upper(VARCHAR-REQUIRED)]", jar2))));
    thread1.start();
    thread2.start();
    latch2.await();
    latch1.countDown();
    thread1.join();
    thread2.join();
    DataChangeVersion version = new DataChangeVersion();
    Registry registry = remoteFunctionRegistry.getRegistry(version);
    assertEquals("Remote registry version should match", 3, version.getVersion());
    List<Jar> actualJars = registry.getJarList();
    List<String> expectedJars = Lists.newArrayList(jar1, jar2);
    assertEquals("Only one jar should be registered", 2, actualJars.size());
    for (Jar jar : actualJars) {
        assertTrue("Jar should be present in remote function registry", expectedJars.contains(jar.getName()));
    }
    verify(remoteFunctionRegistry, times(3)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
}
Also used : RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) Jar(org.apache.drill.exec.proto.UserBitShared.Jar) LocalFunctionRegistry(org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) Registry(org.apache.drill.exec.proto.UserBitShared.Registry) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) CountDownLatch(java.util.concurrent.CountDownLatch) DataChangeVersion(org.apache.drill.exec.store.sys.store.DataChangeVersion) SlowTest(org.apache.drill.categories.SlowTest) SqlFunctionTest(org.apache.drill.categories.SqlFunctionTest) Test(org.junit.Test)

Example 20 with DataChangeVersion

use of org.apache.drill.exec.store.sys.store.DataChangeVersion in project drill by apache.

the class TestDynamicUDFSupport method testSuccessfulUnregistrationAfterSeveralRetryAttempts.

@Test
public void testSuccessfulUnregistrationAfterSeveralRetryAttempts() throws Exception {
    RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
    copyDefaultJarsToStagingArea();
    test("create function using jar '%s'", defaultBinaryJar);
    reset(remoteFunctionRegistry);
    doThrow(new VersionMismatchException("Version mismatch detected", 1)).doThrow(new VersionMismatchException("Version mismatch detected", 1)).doCallRealMethod().when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    String summary = "The following UDFs in jar %s have been unregistered:\n" + "[custom_lower(VARCHAR-REQUIRED)]";
    testBuilder().sqlQuery("drop function using jar '%s'", defaultBinaryJar).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format(summary, defaultBinaryJar)).go();
    verify(remoteFunctionRegistry, times(3)).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
    FileSystem fs = remoteFunctionRegistry.getFs();
    assertFalse("Registry area should be empty", fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
    assertEquals("Registry should be empty", remoteFunctionRegistry.getRegistry(new DataChangeVersion()).getJarList().size(), 0);
}
Also used : RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) FileSystem(org.apache.hadoop.fs.FileSystem) LocalFunctionRegistry(org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) RemoteFunctionRegistry(org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry) Registry(org.apache.drill.exec.proto.UserBitShared.Registry) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) DataChangeVersion(org.apache.drill.exec.store.sys.store.DataChangeVersion) VersionMismatchException(org.apache.drill.exec.exception.VersionMismatchException) SlowTest(org.apache.drill.categories.SlowTest) SqlFunctionTest(org.apache.drill.categories.SqlFunctionTest) Test(org.junit.Test)

Aggregations

DataChangeVersion (org.apache.drill.exec.store.sys.store.DataChangeVersion)43 Test (org.junit.Test)34 RemoteFunctionRegistry (org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry)28 Registry (org.apache.drill.exec.proto.UserBitShared.Registry)25 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)24 FunctionImplementationRegistry (org.apache.drill.exec.expr.fn.FunctionImplementationRegistry)23 LocalFunctionRegistry (org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry)21 VersionMismatchException (org.apache.drill.exec.exception.VersionMismatchException)19 SlowTest (org.apache.drill.categories.SlowTest)16 SqlFunctionTest (org.apache.drill.categories.SqlFunctionTest)16 Matchers.anyString (org.mockito.Matchers.anyString)16 Path (java.nio.file.Path)11 HadoopUtils.hadoopToJavaPath (org.apache.drill.test.HadoopUtils.hadoopToJavaPath)11 FileSystem (org.apache.hadoop.fs.FileSystem)11 Jar (org.apache.drill.exec.proto.UserBitShared.Jar)10 DrillRuntimeException (org.apache.drill.common.exceptions.DrillRuntimeException)9 IOException (java.io.IOException)8 Path (org.apache.hadoop.fs.Path)8 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)8 CountDownLatch (java.util.concurrent.CountDownLatch)6