Examples with Partition io.trino.plugin.hive.metastore.Partition used on opensource projects

Example 16 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class ThriftMetastoreUtil method fromMetastoreApiPartition.

public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition, List<FieldSchema> schema) {
    StorageDescriptor storageDescriptor = partition.getSd();
    if (storageDescriptor == null) {
        throw new TrinoException(HIVE_INVALID_METADATA, "Partition does not contain a storage descriptor: " + partition);
    }
    Partition.Builder partitionBuilder = Partition.builder().setDatabaseName(partition.getDbName()).setTableName(partition.getTableName()).setValues(partition.getValues()).setColumns(schema.stream().map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema).collect(toImmutableList())).setParameters(partition.getParameters());
    // TODO is bucketing_version set on partition level??
    fromMetastoreApiStorageDescriptor(partition.getParameters(), storageDescriptor, partitionBuilder.getStorageBuilder(), format("%s.%s", partition.getTableName(), partition.getValues()));
    return partitionBuilder.build();
}

Example 17 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class AbstractTestHive method eraseStatistics.

private void eraseStatistics(SchemaTableName schemaTableName) {
    HiveMetastore metastoreClient = getMetastoreClient();
    metastoreClient.updateTableStatistics(schemaTableName.getSchemaName(), schemaTableName.getTableName(), NO_ACID_TRANSACTION, statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()));
    Table table = metastoreClient.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    List<String> partitionColumns = table.getPartitionColumns().stream().map(Column::getName).collect(toImmutableList());
    if (!table.getPartitionColumns().isEmpty()) {
        List<String> partitionNames = metastoreClient.getPartitionNamesByFilter(schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionColumns, TupleDomain.all()).orElse(ImmutableList.of());
        List<Partition> partitions = metastoreClient.getPartitionsByNames(table, partitionNames).entrySet().stream().map(Map.Entry::getValue).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
        for (Partition partition : partitions) {
            metastoreClient.updatePartitionStatistics(table, makePartName(partitionColumns, partition.getValues()), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()));
        }
    }
}

Example 18 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class BaseTestHiveOnDataLake method renamePartitionResourcesOutsideTrino.

private void renamePartitionResourcesOutsideTrino(String tableName, String partitionColumn, String regionKey) {
    String partitionName = format("%s=%s", partitionColumn, regionKey);
    String partitionS3KeyPrefix = format("%s/%s/%s", HIVE_TEST_SCHEMA, tableName, partitionName);
    String renamedPartitionSuffix = "CP";
    // Copy whole partition to new location
    AmazonS3 amazonS3 = dockerizedS3DataLake.getS3Client();
    amazonS3.listObjects(bucketName).getObjectSummaries().forEach(object -> {
        String objectKey = object.getKey();
        if (objectKey.startsWith(partitionS3KeyPrefix)) {
            String fileName = objectKey.substring(objectKey.lastIndexOf('/'));
            String destinationKey = partitionS3KeyPrefix + renamedPartitionSuffix + fileName;
            amazonS3.copyObject(bucketName, objectKey, bucketName, destinationKey);
        }
    });
    // Delete old partition and update metadata to point to location of new copy
    Table hiveTable = metastoreClient.getTable(HIVE_TEST_SCHEMA, tableName).get();
    Partition hivePartition = metastoreClient.getPartition(hiveTable, List.of(regionKey)).get();
    Map<String, PartitionStatistics> partitionStatistics = metastoreClient.getPartitionStatistics(hiveTable, List.of(hivePartition));
    metastoreClient.dropPartition(HIVE_TEST_SCHEMA, tableName, List.of(regionKey), true);
    metastoreClient.addPartitions(HIVE_TEST_SCHEMA, tableName, List.of(new PartitionWithStatistics(Partition.builder(hivePartition).withStorage(builder -> builder.setLocation(hivePartition.getStorage().getLocation() + renamedPartitionSuffix)).build(), partitionName, partitionStatistics.get(partitionName))));
}

Example 19 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class TestCachingHiveMetastore method testInvalidGetPartitionsByNames.

@Test
public void testInvalidGetPartitionsByNames() {
    Table table = metastore.getTable(TEST_DATABASE, TEST_TABLE).get();
    Map<String, Optional<Partition>> partitionsByNames = metastore.getPartitionsByNames(table, ImmutableList.of(BAD_PARTITION));
    assertEquals(partitionsByNames.size(), 1);
    Optional<Partition> onlyElement = Iterables.getOnlyElement(partitionsByNames.values());
    assertFalse(onlyElement.isPresent());
}

Example 20 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class TestCachingHiveMetastore method testGetPartitionStatistics.

@Test
public void testGetPartitionStatistics() {
    assertEquals(mockClient.getAccessCount(), 0);
    Table table = metastore.getTable(TEST_DATABASE, TEST_TABLE).get();
    assertEquals(mockClient.getAccessCount(), 1);
    Partition partition = metastore.getPartition(table, TEST_PARTITION_VALUES1).get();
    assertEquals(mockClient.getAccessCount(), 2);
    assertEquals(metastore.getPartitionStatistics(table, ImmutableList.of(partition)), ImmutableMap.of(TEST_PARTITION1, TEST_STATS));
    assertEquals(mockClient.getAccessCount(), 3);
    assertEquals(metastore.getPartitionStatisticsStats().getRequestCount(), 1);
    assertEquals(metastore.getPartitionStatisticsStats().getHitRate(), 0.0);
    assertEquals(metastore.getTableStats().getRequestCount(), 3);
    assertEquals(metastore.getTableStats().getHitRate(), 2.0 / 3);
    assertEquals(metastore.getPartitionStats().getRequestCount(), 2);
    assertEquals(metastore.getPartitionStats().getHitRate(), 0.5);
}