Examples with ColumnIndexID org.apache.hudi.common.util.hash.ColumnIndexID used on opensource projects

Example 1 with ColumnIndexID

use of org.apache.hudi.common.util.hash.ColumnIndexID in project hudi by apache.

the class HoodieMetadataPayload method getColumnStatsIndexKey.

/**
 * Get column stats index key from the column range metadata.
 *
 * @param partitionName       - Partition name
 * @param columnRangeMetadata -  Column range metadata
 * @return Column stats index key
 */
public static String getColumnStatsIndexKey(String partitionName, HoodieColumnRangeMetadata<Comparable> columnRangeMetadata) {
    final PartitionIndexID partitionIndexID = new PartitionIndexID(partitionName);
    final FileIndexID fileIndexID = new FileIndexID(new Path(columnRangeMetadata.getFilePath()).getName());
    final ColumnIndexID columnIndexID = new ColumnIndexID(columnRangeMetadata.getColumnName());
    return getColumnStatsIndexKey(partitionIndexID, fileIndexID, columnIndexID);
}

Example 2 with ColumnIndexID

use of org.apache.hudi.common.util.hash.ColumnIndexID in project hudi by apache.

the class BaseTableMetadata method getColumnStats.

@Override
public Map<Pair<String, String>, HoodieMetadataColumnStats> getColumnStats(final List<Pair<String, String>> partitionNameFileNameList, final String columnName) throws HoodieMetadataException {
    if (!isColumnStatsIndexEnabled) {
        LOG.error("Metadata column stats index is disabled!");
        return Collections.emptyMap();
    }
    Map<String, Pair<String, String>> columnStatKeyToFileNameMap = new HashMap<>();
    TreeSet<String> sortedKeys = new TreeSet<>();
    final ColumnIndexID columnIndexID = new ColumnIndexID(columnName);
    for (Pair<String, String> partitionNameFileNamePair : partitionNameFileNameList) {
        final String columnStatsIndexKey = HoodieMetadataPayload.getColumnStatsIndexKey(new PartitionIndexID(partitionNameFileNamePair.getLeft()), new FileIndexID(partitionNameFileNamePair.getRight()), columnIndexID);
        sortedKeys.add(columnStatsIndexKey);
        columnStatKeyToFileNameMap.put(columnStatsIndexKey, partitionNameFileNamePair);
    }
    List<String> columnStatKeys = new ArrayList<>(sortedKeys);
    HoodieTimer timer = new HoodieTimer().startTimer();
    List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> hoodieRecordList = getRecordsByKeys(columnStatKeys, MetadataPartitionType.COLUMN_STATS.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer()));
    Map<Pair<String, String>, HoodieMetadataColumnStats> fileToColumnStatMap = new HashMap<>();
    for (final Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry : hoodieRecordList) {
        if (entry.getRight().isPresent()) {
            final Option<HoodieMetadataColumnStats> columnStatMetadata = entry.getRight().get().getData().getColumnStatMetadata();
            if (columnStatMetadata.isPresent()) {
                if (!columnStatMetadata.get().getIsDeleted()) {
                    ValidationUtils.checkState(columnStatKeyToFileNameMap.containsKey(entry.getLeft()));
                    final Pair<String, String> partitionFileNamePair = columnStatKeyToFileNameMap.get(entry.getLeft());
                    ValidationUtils.checkState(!fileToColumnStatMap.containsKey(partitionFileNamePair));
                    fileToColumnStatMap.put(partitionFileNamePair, columnStatMetadata.get());
                }
            } else {
                LOG.error("Meta index column stats missing for: " + entry.getLeft());
            }
        }
    }
    return fileToColumnStatMap;
}