Search in sources :

Example 1 with ColumnConverter

use of com.facebook.presto.hive.ColumnConverter in project presto by prestodb.

the class TestCachingHiveMetastore method testCachingWithPartitionVersioning.

@Test
public void testCachingWithPartitionVersioning() {
    MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
    MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
    PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity());
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    CachingHiveMetastore partitionCachingEnabledmetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 0.0);
    assertEquals(mockClient.getAccessCount(), 0);
    assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
    assertEquals(mockClient.getAccessCount(), 1);
    assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
    // Assert that we did not hit cache
    assertEquals(mockClient.getAccessCount(), 2);
    // Select all of the available partitions and load them into the cache
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
    assertEquals(mockClient.getAccessCount(), 3);
    // Now if we fetch any or both of them, they should hit the cache
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).size(), 1);
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).size(), 1);
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
    assertEquals(mockClient.getAccessCount(), 3);
    // This call should NOT invalidate the partition cache because partition version is same as before
    assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
    assertEquals(mockClient.getAccessCount(), 4);
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
    // Assert that its a cache hit
    assertEquals(mockClient.getAccessCount(), 4);
    assertInvalidateCache(new MockPartitionMutator(version -> version + 1));
    assertInvalidateCache(new MockPartitionMutator(version -> version - 1));
}
Also used : MockHiveMetastoreClient(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) Iterables(com.google.common.collect.Iterables) UnaryOperator.identity(java.util.function.UnaryOperator.identity) MockHiveMetastoreClient(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) Function(java.util.function.Function) TEST_METASTORE_CONTEXT(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_METASTORE_CONTEXT) Duration(io.airlift.units.Duration) TEST_TABLE(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_TABLE) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) MetastoreCacheScope(com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) MetastoreClientConfig(com.facebook.presto.hive.MetastoreClientConfig) HiveMetastoreClient(com.facebook.presto.hive.metastore.thrift.HiveMetastoreClient) Assert.assertFalse(org.testng.Assert.assertFalse) MockHiveMetastore(com.facebook.presto.hive.MockHiveMetastore) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) TEST_PARTITION_VALUES2(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_PARTITION_VALUES2) PartitionMutator(com.facebook.presto.hive.PartitionMutator) ImmutableMap(com.google.common.collect.ImmutableMap) TEST_PARTITION_VALUES1(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_PARTITION_VALUES1) BeforeMethod(org.testng.annotations.BeforeMethod) Assert.assertNotNull(org.testng.Assert.assertNotNull) TEST_DATABASE(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_DATABASE) TimeUnit(java.util.concurrent.TimeUnit) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) ColumnConverter(com.facebook.presto.hive.ColumnConverter) Builder(com.facebook.presto.hive.metastore.Partition.Builder) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) Optional(java.util.Optional) BAD_DATABASE(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.BAD_DATABASE) TEST_ROLES(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_ROLES) Assert.assertTrue(org.testng.Assert.assertTrue) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) PARTITION_VERSION(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.PARTITION_VERSION) ThriftHiveMetastoreStats(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastoreStats) TEST_PARTITION1(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_PARTITION1) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) TEST_PARTITION2(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_PARTITION2) PartitionMutator(com.facebook.presto.hive.PartitionMutator) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Duration(io.airlift.units.Duration) ColumnConverter(com.facebook.presto.hive.ColumnConverter) MockHiveMetastore(com.facebook.presto.hive.MockHiveMetastore) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) Test(org.testng.annotations.Test)

Example 2 with ColumnConverter

use of com.facebook.presto.hive.ColumnConverter in project presto by prestodb.

the class TestCachingHiveMetastore method testPartitionCacheValidation.

@Test
public void testPartitionCacheValidation() {
    MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
    MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
    PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity());
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    CachingHiveMetastore partitionCacheVerificationEnabledMetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 100.0);
    // Warmup the cache
    partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2));
    // Each of the following calls will be a cache hit and verification will be done.
    partitionCacheVerificationEnabledMetastore.getPartition(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, TEST_PARTITION_VALUES1);
    partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1));
    partitionCacheVerificationEnabledMetastore.getPartition(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, TEST_PARTITION_VALUES2);
    partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2));
}
Also used : MockHiveMetastoreClient(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient) PartitionMutator(com.facebook.presto.hive.PartitionMutator) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Duration(io.airlift.units.Duration) ColumnConverter(com.facebook.presto.hive.ColumnConverter) MockHiveMetastore(com.facebook.presto.hive.MockHiveMetastore) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) Test(org.testng.annotations.Test)

Example 3 with ColumnConverter

use of com.facebook.presto.hive.ColumnConverter in project presto by prestodb.

the class TestCachingHiveMetastore method setUp.

@BeforeMethod
public void setUp() {
    mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(mockHiveCluster, new MetastoreClientConfig());
    PartitionMutator hivePartitionMutator = new HivePartitionMutator();
    metastore = new CachingHiveMetastore(new BridgingHiveMetastore(thriftHiveMetastore, hivePartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, false, MetastoreCacheScope.ALL, 0.0);
    stats = thriftHiveMetastore.getStats();
}
Also used : MockHiveMetastoreClient(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient) PartitionMutator(com.facebook.presto.hive.PartitionMutator) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Duration(io.airlift.units.Duration) ColumnConverter(com.facebook.presto.hive.ColumnConverter) MetastoreClientConfig(com.facebook.presto.hive.MetastoreClientConfig) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 4 with ColumnConverter

use of com.facebook.presto.hive.ColumnConverter in project presto by prestodb.

the class TestCachingHiveMetastore method assertInvalidateCache.

private void assertInvalidateCache(MockPartitionMutator partitionMutator) {
    MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
    MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    CachingHiveMetastore partitionCachingEnabledmetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, partitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 0.0);
    int clientAccessCount = 0;
    for (int i = 0; i < 100; i++) {
        assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
        assertEquals(mockClient.getAccessCount(), ++clientAccessCount);
        assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
        // Assert that we did not hit cache
        assertEquals(mockClient.getAccessCount(), ++clientAccessCount);
    }
}
Also used : MockHiveMetastoreClient(com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Duration(io.airlift.units.Duration) ColumnConverter(com.facebook.presto.hive.ColumnConverter) MockHiveMetastore(com.facebook.presto.hive.MockHiveMetastore) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore)

Aggregations

ColumnConverter (com.facebook.presto.hive.ColumnConverter)4 BridgingHiveMetastore (com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore)4 MockHiveMetastoreClient (com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient)4 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)4 Duration (io.airlift.units.Duration)4 MockHiveMetastore (com.facebook.presto.hive.MockHiveMetastore)3 PartitionMutator (com.facebook.presto.hive.PartitionMutator)3 MetastoreClientConfig (com.facebook.presto.hive.MetastoreClientConfig)2 ThriftHiveMetastore (com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore)2 BeforeMethod (org.testng.annotations.BeforeMethod)2 Threads.daemonThreadsNamed (com.facebook.airlift.concurrent.Threads.daemonThreadsNamed)1 MetastoreCacheScope (com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope)1 Builder (com.facebook.presto.hive.metastore.Partition.Builder)1 HiveCluster (com.facebook.presto.hive.metastore.thrift.HiveCluster)1 HiveMetastoreClient (com.facebook.presto.hive.metastore.thrift.HiveMetastoreClient)1 BAD_DATABASE (com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.BAD_DATABASE)1 PARTITION_VERSION (com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.PARTITION_VERSION)1 TEST_DATABASE (com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_DATABASE)1 TEST_METASTORE_CONTEXT (com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_METASTORE_CONTEXT)1 TEST_PARTITION1 (com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_PARTITION1)1