Examples with HoodieMetadataBloomFilter org.apache.hudi.avro.model.HoodieMetadataBloomFilter used on opensource projects

Example 1 with HoodieMetadataBloomFilter

use of org.apache.hudi.avro.model.HoodieMetadataBloomFilter in project hudi by apache.

the class HoodieMetadataPayload method createBloomFilterMetadataRecord.

/**
 * Create bloom filter metadata record.
 *
 * @param partitionName - Partition name
 * @param baseFileName  - Base file name for which the bloom filter needs to persisted
 * @param timestamp     - Instant timestamp responsible for this record
 * @param bloomFilter   - Bloom filter for the File
 * @param isDeleted     - Is the bloom filter no more valid
 * @return Metadata payload containing the fileID and its bloom filter record
 */
public static HoodieRecord<HoodieMetadataPayload> createBloomFilterMetadataRecord(final String partitionName, final String baseFileName, final String timestamp, final String bloomFilterType, final ByteBuffer bloomFilter, final boolean isDeleted) {
    checkArgument(!baseFileName.contains(Path.SEPARATOR) && FSUtils.isBaseFile(new Path(baseFileName)), "Invalid base file '" + baseFileName + "' for MetaIndexBloomFilter!");
    final String bloomFilterIndexKey = new PartitionIndexID(partitionName).asBase64EncodedString().concat(new FileIndexID(baseFileName).asBase64EncodedString());
    HoodieKey key = new HoodieKey(bloomFilterIndexKey, MetadataPartitionType.BLOOM_FILTERS.getPartitionPath());
    HoodieMetadataBloomFilter metadataBloomFilter = new HoodieMetadataBloomFilter(bloomFilterType, timestamp, bloomFilter, isDeleted);
    HoodieMetadataPayload metadataPayload = new HoodieMetadataPayload(key.getRecordKey(), metadataBloomFilter);
    return new HoodieAvroRecord<>(key, metadataPayload);
}

Example 2 with HoodieMetadataBloomFilter

use of org.apache.hudi.avro.model.HoodieMetadataBloomFilter in project hudi by apache.

the class BaseTableMetadata method getBloomFilters.

@Override
public Map<Pair<String, String>, BloomFilter> getBloomFilters(final List<Pair<String, String>> partitionNameFileNameList) throws HoodieMetadataException {
    if (!isBloomFilterIndexEnabled) {
        LOG.error("Metadata bloom filter index is disabled!");
        return Collections.emptyMap();
    }
    if (partitionNameFileNameList.isEmpty()) {
        return Collections.emptyMap();
    }
    HoodieTimer timer = new HoodieTimer().startTimer();
    Set<String> partitionIDFileIDSortedStrings = new TreeSet<>();
    Map<String, Pair<String, String>> fileToKeyMap = new HashMap<>();
    partitionNameFileNameList.forEach(partitionNameFileNamePair -> {
        final String bloomFilterIndexKey = HoodieMetadataPayload.getBloomFilterIndexKey(new PartitionIndexID(partitionNameFileNamePair.getLeft()), new FileIndexID(partitionNameFileNamePair.getRight()));
        partitionIDFileIDSortedStrings.add(bloomFilterIndexKey);
        fileToKeyMap.put(bloomFilterIndexKey, partitionNameFileNamePair);
    });
    List<String> partitionIDFileIDStrings = new ArrayList<>(partitionIDFileIDSortedStrings);
    List<Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>>> hoodieRecordList = getRecordsByKeys(partitionIDFileIDStrings, MetadataPartitionType.BLOOM_FILTERS.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR, (timer.endTimer() / partitionIDFileIDStrings.size())));
    Map<Pair<String, String>, BloomFilter> partitionFileToBloomFilterMap = new HashMap<>();
    for (final Pair<String, Option<HoodieRecord<HoodieMetadataPayload>>> entry : hoodieRecordList) {
        if (entry.getRight().isPresent()) {
            final Option<HoodieMetadataBloomFilter> bloomFilterMetadata = entry.getRight().get().getData().getBloomFilterMetadata();
            if (bloomFilterMetadata.isPresent()) {
                if (!bloomFilterMetadata.get().getIsDeleted()) {
                    ValidationUtils.checkState(fileToKeyMap.containsKey(entry.getLeft()));
                    final ByteBuffer bloomFilterByteBuffer = bloomFilterMetadata.get().getBloomFilter();
                    final String bloomFilterType = bloomFilterMetadata.get().getType();
                    final BloomFilter bloomFilter = BloomFilterFactory.fromString(StandardCharsets.UTF_8.decode(bloomFilterByteBuffer).toString(), bloomFilterType);
                    partitionFileToBloomFilterMap.put(fileToKeyMap.get(entry.getLeft()), bloomFilter);
                }
            } else {
                LOG.error("Meta index bloom filter missing for: " + fileToKeyMap.get(entry.getLeft()));
            }
        }
    }
    return partitionFileToBloomFilterMap;
}