Examples with Partition org.apache.hadoop.hive.metastore.api.Partition used on opensource projects

Example 91 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project flink by apache.

the class HiveCatalog method getPartitionColumnStatistics.

@Override
public CatalogColumnStatistics getPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException {
    try {
        Partition partition = getHivePartition(tablePath, partitionSpec);
        Table hiveTable = getHiveTable(tablePath);
        String partName = getEscapedPartitionName(tablePath, partitionSpec, hiveTable);
        List<String> partNames = new ArrayList<>();
        partNames.add(partName);
        Map<String, List<ColumnStatisticsObj>> partitionColumnStatistics = client.getPartitionColumnStatistics(partition.getDbName(), partition.getTableName(), partNames, getFieldNames(partition.getSd().getCols()));
        List<ColumnStatisticsObj> columnStatisticsObjs = partitionColumnStatistics.get(partName);
        if (columnStatisticsObjs != null && !columnStatisticsObjs.isEmpty()) {
            return new CatalogColumnStatistics(HiveStatsUtil.createCatalogColumnStats(columnStatisticsObjs, hiveVersion));
        } else {
            return CatalogColumnStatistics.UNKNOWN;
        }
    } catch (TableNotExistException | PartitionSpecInvalidException e) {
        throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
    } catch (TException e) {
        throw new CatalogException(String.format("Failed to get table stats of table %s 's partition %s", tablePath.getFullName(), String.valueOf(partitionSpec)), e);
    }
}

Example 92 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project flink by apache.

the class HiveCatalog method getPartition.

@Override
public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws PartitionNotExistException, CatalogException {
    checkNotNull(tablePath, "Table path cannot be null");
    checkNotNull(partitionSpec, "CatalogPartitionSpec cannot be null");
    try {
        Partition hivePartition = getHivePartition(tablePath, partitionSpec);
        Map<String, String> properties = hivePartition.getParameters();
        properties.put(SqlCreateHiveTable.TABLE_LOCATION_URI, hivePartition.getSd().getLocation());
        String comment = properties.remove(HiveCatalogConfig.COMMENT);
        return new CatalogPartitionImpl(properties, comment);
    } catch (NoSuchObjectException | MetaException | TableNotExistException | PartitionSpecInvalidException e) {
        throw new PartitionNotExistException(getName(), tablePath, partitionSpec, e);
    } catch (TException e) {
        throw new CatalogException(String.format("Failed to get partition %s of table %s", partitionSpec, tablePath), e);
    }
}

Example 93 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project flink by apache.

the class HivePartitionUtils method getAllPartitions.

/**
 * Returns all HiveTablePartitions of a hive table, returns single HiveTablePartition if the
 * hive table is not partitioned.
 */
public static List<HiveTablePartition> getAllPartitions(JobConf jobConf, String hiveVersion, ObjectPath tablePath, List<String> partitionColNames, List<Map<String, String>> remainingPartitions) {
    List<HiveTablePartition> allHivePartitions = new ArrayList<>();
    try (HiveMetastoreClientWrapper client = HiveMetastoreClientFactory.create(HiveConfUtils.create(jobConf), hiveVersion)) {
        String dbName = tablePath.getDatabaseName();
        String tableName = tablePath.getObjectName();
        Table hiveTable = client.getTable(dbName, tableName);
        Properties tableProps = HiveReflectionUtils.getTableMetadata(HiveShimLoader.loadHiveShim(hiveVersion), hiveTable);
        if (partitionColNames != null && partitionColNames.size() > 0) {
            List<Partition> partitions = new ArrayList<>();
            if (remainingPartitions != null) {
                for (Map<String, String> spec : remainingPartitions) {
                    partitions.add(client.getPartition(dbName, tableName, partitionSpecToValues(spec, partitionColNames)));
                }
            } else {
                partitions.addAll(client.listPartitions(dbName, tableName, (short) -1));
            }
            for (Partition partition : partitions) {
                HiveTablePartition hiveTablePartition = toHiveTablePartition(partitionColNames, tableProps, partition);
                allHivePartitions.add(hiveTablePartition);
            }
        } else {
            allHivePartitions.add(new HiveTablePartition(hiveTable.getSd(), tableProps));
        }
    } catch (TException e) {
        throw new FlinkHiveException("Failed to collect all partitions from hive metaStore", e);
    }
    return allHivePartitions;
}

Example 94 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project flink by apache.

the class HiveCatalog method alterPartitionColumnStatistics.

@Override
public void alterPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException {
    try {
        Partition hivePartition = getHivePartition(tablePath, partitionSpec);
        Table hiveTable = getHiveTable(tablePath);
        String partName = getEscapedPartitionName(tablePath, partitionSpec, hiveTable);
        client.updatePartitionColumnStatistics(HiveStatsUtil.createPartitionColumnStats(hivePartition, partName, columnStatistics.getColumnStatisticsData(), hiveVersion));
    } catch (TableNotExistException | PartitionSpecInvalidException e) {
        if (!ignoreIfNotExists) {
            throw new PartitionNotExistException(getName(), tablePath, partitionSpec, e);
        }
    } catch (TException e) {
        throw new CatalogException(String.format("Failed to alter table column stats of table %s 's partition %s", tablePath.getFullName(), String.valueOf(partitionSpec)), e);
    }
}

Example 95 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project flink by apache.

the class HiveCatalog method alterPartitionStatistics.

@Override
public void alterPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogTableStatistics partitionStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException {
    try {
        Partition hivePartition = getHivePartition(tablePath, partitionSpec);
        // Set table stats
        if (statsChanged(partitionStatistics, hivePartition.getParameters())) {
            updateStats(partitionStatistics, hivePartition.getParameters());
            client.alter_partition(tablePath.getDatabaseName(), tablePath.getObjectName(), hivePartition);
        }
    } catch (TableNotExistException | PartitionSpecInvalidException e) {
        throw new PartitionNotExistException(getName(), tablePath, partitionSpec, e);
    } catch (TException e) {
        throw new CatalogException(String.format("Failed to alter table stats of table %s 's partition %s", tablePath.getFullName(), String.valueOf(partitionSpec)), e);
    }
}