Examples with PartitionMutator com.facebook.presto.hive.PartitionMutator used on opensource projects

Example 1 with PartitionMutator

use of com.facebook.presto.hive.PartitionMutator in project presto by prestodb.

the class TestCachingHiveMetastore method testCachingWithPartitionVersioning.

@Test
public void testCachingWithPartitionVersioning() {
    MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
    MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
    PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity());
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    CachingHiveMetastore partitionCachingEnabledmetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 0.0);
    assertEquals(mockClient.getAccessCount(), 0);
    assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
    assertEquals(mockClient.getAccessCount(), 1);
    assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
    // Assert that we did not hit cache
    assertEquals(mockClient.getAccessCount(), 2);
    // Select all of the available partitions and load them into the cache
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
    assertEquals(mockClient.getAccessCount(), 3);
    // Now if we fetch any or both of them, they should hit the cache
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).size(), 1);
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).size(), 1);
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
    assertEquals(mockClient.getAccessCount(), 3);
    // This call should NOT invalidate the partition cache because partition version is same as before
    assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS);
    assertEquals(mockClient.getAccessCount(), 4);
    assertEquals(partitionCachingEnabledmetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2);
    // Assert that its a cache hit
    assertEquals(mockClient.getAccessCount(), 4);
    assertInvalidateCache(new MockPartitionMutator(version -> version + 1));
    assertInvalidateCache(new MockPartitionMutator(version -> version - 1));
}

Example 2 with PartitionMutator

use of com.facebook.presto.hive.PartitionMutator in project presto by prestodb.

the class BridgingHiveMetastore method getPartitionsByNames.

@Override
public Map<String, Optional<Partition>> getPartitionsByNames(MetastoreContext metastoreContext, String databaseName, String tableName, List<String> partitionNames) {
    requireNonNull(partitionNames, "partitionNames is null");
    if (partitionNames.isEmpty()) {
        return ImmutableMap.of();
    }
    Map<String, List<String>> partitionNameToPartitionValuesMap = partitionNames.stream().collect(Collectors.toMap(identity(), MetastoreUtil::toPartitionValues));
    Map<List<String>, Partition> partitionValuesToPartitionMap = delegate.getPartitionsByNames(metastoreContext, databaseName, tableName, partitionNames).stream().map(partition -> fromMetastoreApiPartition(partition, partitionMutator, metastoreContext.getColumnConverter())).collect(Collectors.toMap(Partition::getValues, identity()));
    ImmutableMap.Builder<String, Optional<Partition>> resultBuilder = ImmutableMap.builder();
    for (Map.Entry<String, List<String>> entry : partitionNameToPartitionValuesMap.entrySet()) {
        Partition partition = partitionValuesToPartitionMap.get(entry.getValue());
        resultBuilder.put(entry.getKey(), Optional.ofNullable(partition));
    }
    return resultBuilder.build();
}

Example 3 with PartitionMutator

use of com.facebook.presto.hive.PartitionMutator in project presto by prestodb.

the class TestCachingHiveMetastore method testPartitionCacheValidation.

@Test
public void testPartitionCacheValidation() {
    MockHiveMetastoreClient mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s")));
    MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster);
    PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity());
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    CachingHiveMetastore partitionCacheVerificationEnabledMetastore = new CachingHiveMetastore(new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, true, MetastoreCacheScope.PARTITION, 100.0);
    // Warmup the cache
    partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2));
    // Each of the following calls will be a cache hit and verification will be done.
    partitionCacheVerificationEnabledMetastore.getPartition(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, TEST_PARTITION_VALUES1);
    partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1));
    partitionCacheVerificationEnabledMetastore.getPartition(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, TEST_PARTITION_VALUES2);
    partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2));
}

Example 4 with PartitionMutator

use of com.facebook.presto.hive.PartitionMutator in project presto by prestodb.

the class TestCachingHiveMetastore method setUp.

@BeforeMethod
public void setUp() {
    mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    ColumnConverter hiveColumnConverter = new HiveColumnConverter();
    ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(mockHiveCluster, new MetastoreClientConfig());
    PartitionMutator hivePartitionMutator = new HivePartitionMutator();
    metastore = new CachingHiveMetastore(new BridgingHiveMetastore(thriftHiveMetastore, hivePartitionMutator), executor, false, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, false, MetastoreCacheScope.ALL, 0.0);
    stats = thriftHiveMetastore.getStats();
}