use of com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient in project presto by prestodb.
the class TestCachingHiveMetastore method testCachingWithPartitionVersioning.
@Test
public void testCachingWithPartitionVersioning() {
MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity());
ColumnConverter hiveColumnConverter = new HiveColumnConverter();
CachingHiveMetastore partitionCachingEnabledmetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 0.0);
assertEquals(mockClient.getAccessCount(), 0);
assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
assertEquals(mockClient.getAccessCount(), 1);
assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
// Assert that we did not hit cache
assertEquals(mockClient.getAccessCount(), 2);
// Select all of the available partitions and load them into the cache
assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
assertEquals(mockClient.getAccessCount(), 3);
// Now if we fetch any or both of them, they should hit the cache
assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).size(), 1);
assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).size(), 1);
assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
assertEquals(mockClient.getAccessCount(), 3);
// This call should NOT invalidate the partition cache because partition version is same as before
assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
assertEquals(mockClient.getAccessCount(), 4);
assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
// Assert that its a cache hit
assertEquals(mockClient.getAccessCount(), 4);
assertInvalidateCache(new MockPartitionMutator(version -> version + 1));
assertInvalidateCache(new MockPartitionMutator(version -> version - 1));
}
use of com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient in project presto by prestodb.
the class TestCachingHiveMetastore method testPartitionCacheValidation.
@Test
public void testPartitionCacheValidation() {
MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity());
ColumnConverter hiveColumnConverter = new HiveColumnConverter();
CachingHiveMetastore partitionCacheVerificationEnabledMetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 100.0);
// Warmup the cache
partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2));
// Each of the following calls will be a cache hit and verification will be done.
partitionCacheVerificationEnabledMetastore.getPartition(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, TEST_PARTITION_VALUES1);
partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1));
partitionCacheVerificationEnabledMetastore.getPartition(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, TEST_PARTITION_VALUES2);
partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2));
}
use of com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient in project presto by prestodb.
the class TestCachingHiveMetastore method setUp.
@BeforeMethod
public void setUp() {
mockClient = new MockHiveMetastoreClient();
MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
ColumnConverter hiveColumnConverter = new HiveColumnConverter();
ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(mockHiveCluster, new MetastoreClientConfig());
PartitionMutator hivePartitionMutator = new HivePartitionMutator();
metastore = new CachingHiveMetastore(new BridgingHiveMetastore(thriftHiveMetastore, hivePartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, false, MetastoreCacheScope.ALL, 0.0);
stats = thriftHiveMetastore.getStats();
}
use of com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient in project presto by prestodb.
the class TestCachingHiveMetastore method assertInvalidateCache.
private void assertInvalidateCache(MockPartitionMutator partitionMutator) {
MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
ColumnConverter hiveColumnConverter = new HiveColumnConverter();
CachingHiveMetastore partitionCachingEnabledmetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, partitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 0.0);
int clientAccessCount = 0;
for (int i = 0; i < 100; i++) {
assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
assertEquals(mockClient.getAccessCount(), ++clientAccessCount);
assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
// Assert that we did not hit cache
assertEquals(mockClient.getAccessCount(), ++clientAccessCount);
}
}
Aggregations