Examples with BloomFilter org.apache.parquet.column.values.bloomfilter.BloomFilter used on opensource projects

Example 6 with BloomFilter

use of org.apache.parquet.column.values.bloomfilter.BloomFilter in project parquet-mr by apache.

the class BloomFilterReader method readBloomFilter.

public BloomFilter readBloomFilter(ColumnChunkMetaData meta) {
    if (cache.containsKey(meta.getPath())) {
        return cache.get(meta.getPath());
    }
    try {
        if (!cache.containsKey(meta.getPath())) {
            BloomFilter bloomFilter = reader.readBloomFilter(meta);
            if (bloomFilter == null) {
                return null;
            }
            cache.put(meta.getPath(), bloomFilter);
        }
        return cache.get(meta.getPath());
    } catch (IOException e) {
        logger.error("Failed to read Bloom filter data", e);
    }
    return null;
}

Example 7 with BloomFilter

use of org.apache.parquet.column.values.bloomfilter.BloomFilter in project parquet-mr by apache.

the class BloomFilterImpl method visit.

@Override
public <T extends Comparable<T>> Boolean visit(Operators.Eq<T> eq) {
    T value = eq.getValue();
    if (value == null) {
        // could check the column stats, but the StatisticsFilter is responsible
        return BLOCK_MIGHT_MATCH;
    }
    Operators.Column<T> filterColumn = eq.getColumn();
    ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath());
    if (meta == null) {
        // must be non-null because of the above check.
        return BLOCK_CANNOT_MATCH;
    }
    try {
        BloomFilter bloomFilter = bloomFilterReader.readBloomFilter(meta);
        if (bloomFilter != null && !bloomFilter.findHash(bloomFilter.hash(value))) {
            return BLOCK_CANNOT_MATCH;
        }
    } catch (RuntimeException e) {
        LOG.warn(e.getMessage());
        return BLOCK_MIGHT_MATCH;
    }
    return BLOCK_MIGHT_MATCH;
}

Example 8 with BloomFilter

use of org.apache.parquet.column.values.bloomfilter.BloomFilter in project parquet-mr by apache.

the class BloomFilterImpl method visit.

@Override
public <T extends Comparable<T>> Boolean visit(Operators.In<T> in) {
    Set<T> values = in.getValues();
    if (values.contains(null)) {
        // could check the column stats, but the StatisticsFilter is responsible
        return BLOCK_MIGHT_MATCH;
    }
    Operators.Column<T> filterColumn = in.getColumn();
    ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath());
    if (meta == null) {
        // must be non-null because of the above check.
        return BLOCK_CANNOT_MATCH;
    }
    BloomFilter bloomFilter = bloomFilterReader.readBloomFilter(meta);
    if (bloomFilter != null) {
        for (T value : values) {
            if (bloomFilter.findHash(bloomFilter.hash(value))) {
                return BLOCK_MIGHT_MATCH;
            }
        }
        return BLOCK_CANNOT_MATCH;
    }
    return BLOCK_MIGHT_MATCH;
}