use of io.prestosql.spi.dynamicfilter.DynamicFilter in project hetu-core by openlookeng.
the class TestHiveUtil method testIsPartitionFilteredWithNonPartitionFilter.
@Test
public void testIsPartitionFilteredWithNonPartitionFilter() {
TypeManager typeManager = new TestingTypeManager();
Set<DynamicFilter> dynamicFilters = new HashSet<>();
List<HivePartitionKey> partitions = new ArrayList<>();
partitions.add(new HivePartitionKey("pt_d", "0"));
partitions.add(new HivePartitionKey("app_id", "10000"));
ColumnHandle nameColumn = new HiveColumnHandle("name", HIVE_STRING, parseTypeSignature(VARCHAR), 0, REGULAR, Optional.empty());
Set nameFilter = new HashSet();
nameFilter.add("Alice");
dynamicFilters.add(new HashSetDynamicFilter("1", nameColumn, nameFilter, DynamicFilter.Type.GLOBAL));
assertFalse(isPartitionFiltered(partitions, ImmutableList.of(dynamicFilters), typeManager), "Should not filter partition if dynamicFilter is on non-partition column");
}
use of io.prestosql.spi.dynamicfilter.DynamicFilter in project hetu-core by openlookeng.
the class DynamicFilterListener method transformDynamicFilter.
private Optional<DynamicFilter> transformDynamicFilter(String cacheKey, Object newDynamicFilter) {
String filterId = cacheKey.split("-")[0];
// Global dynamic filters
if (newDynamicFilter == null) {
LOG.warn("DynamicFilter added to StateStore is null.");
return Optional.empty();
}
DynamicFilter dynamicFilter = null;
if (newDynamicFilter instanceof Set) {
dynamicFilter = DynamicFilterFactory.create(filterId, null, (Set<?>) newDynamicFilter, GLOBAL);
LOG.debug("Got new HashSet DynamicFilter from state store: " + filterId + ", size: " + dynamicFilter.getSize());
} else if (newDynamicFilter instanceof byte[]) {
dynamicFilter = DynamicFilterFactory.create(filterId, null, (byte[]) newDynamicFilter, GLOBAL);
LOG.debug("Got new BloomFilter DynamicFilter from state store: " + filterId + ", size: " + dynamicFilter.getSize());
}
return dynamicFilter == null ? Optional.empty() : Optional.of(dynamicFilter);
}
use of io.prestosql.spi.dynamicfilter.DynamicFilter in project hetu-core by openlookeng.
the class LocalDynamicFiltersCollector method getDynamicFilters.
/**
* This function returns the bloom filters fetched from the state store. To prevent excessive reads from state store,
* it caches fetched bloom filters for re-use
*
* @param tableScan TableScanNode that has DynamicFilter applied
* @return ColumnHandle to DynamicFilter mapping that contains any DynamicFilter that are ready for use
*/
List<Map<ColumnHandle, DynamicFilter>> getDynamicFilters(TableScanNode tableScan) {
Map<Symbol, ColumnHandle> assignments = tableScan.getAssignments();
// Skips symbols irrelevant to this table scan node.
Set<String> columnNames = new HashSet<>();
List<Map<ColumnHandle, DynamicFilter>> resultList = new ArrayList<>();
for (int i = 0; i < context.getDisjunctSize(); i++) {
Map<ColumnHandle, DynamicFilter> result = new HashMap<ColumnHandle, DynamicFilter>();
for (Map.Entry<Symbol, ColumnHandle> entry : assignments.entrySet()) {
final Symbol columnSymbol = entry.getKey();
final ColumnHandle columnHandle = entry.getValue();
try {
columnNames.add(columnHandle.getColumnName());
} catch (NotImplementedException e) {
// ignore this exception, maybe some implementation class not implement the default method.
}
final List<String> filterIds = context.getId(columnSymbol, i);
if (filterIds == null || filterIds.isEmpty()) {
continue;
}
for (String filterId : filterIds) {
// Try to get dynamic filter from local cache first
String cacheKey = createCacheKey(filterId, queryId);
DynamicFilter cachedDynamicFilter = cachedDynamicFilters.get(filterId);
if (cachedDynamicFilter == null) {
cachedDynamicFilter = dynamicFilterCacheManager.getDynamicFilter(cacheKey);
}
if (cachedDynamicFilter != null) {
// Combine multiple dynamic filters for same column handle
DynamicFilter dynamicFilter = result.get(columnHandle);
// Same dynamic filter might be referred in multiple table scans for different columns due multi table joins.
// So clone before setting the columnHandle to avoid race in setting the columnHandle.
cachedDynamicFilter = cachedDynamicFilter.clone();
cachedDynamicFilter.setColumnHandle(columnHandle);
if (dynamicFilter == null) {
dynamicFilter = cachedDynamicFilter;
} else {
dynamicFilter = DynamicFilterFactory.combine(columnHandle, dynamicFilter, cachedDynamicFilter);
}
dynamicFilter.setColumnHandle(columnHandle);
result.put(columnHandle, dynamicFilter);
continue;
}
// Local dynamic filters
if (predicates.containsKey(filterId)) {
Optional<RowExpression> filter = context.getFilter(filterId, i);
Optional<Predicate<List>> filterPredicate = DynamicFilters.createDynamicFilterPredicate(filter);
DynamicFilter dynamicFilter = DynamicFilterFactory.create(filterId, columnHandle, predicates.get(filterId), LOCAL, filterPredicate, filter);
cachedDynamicFilters.put(filterId, dynamicFilter);
result.put(columnHandle, dynamicFilter);
}
}
}
if (!result.isEmpty()) {
resultList.add(result);
}
}
if (isCrossRegionDynamicFilterEnabled(session)) {
if (!metadataOptional.isPresent()) {
return resultList;
}
// check the tableScan is a dc connector table,if a dc table, should consider push down the cross region bloom filter to next cluster
if (!DataCenterUtility.isDCCatalog(metadataOptional.get(), tableScan.getTable().getCatalogName().getCatalogName())) {
return resultList;
}
// stateMap, key is dc-connector-table column name, value is bloomFilter bytes
Map<String, byte[]> newBloomFilterFromStateStoreCache = dynamicFilterCacheManager.getBloomFitler(session.getQueryId().getId() + CROSS_LAYER_DYNAMIC_FILTER);
if (newBloomFilterFromStateStoreCache == null) {
return resultList;
}
// check tableScan contains the stateMap.key, if contains, should push the filter to next cluster
for (Map.Entry<String, byte[]> entry : newBloomFilterFromStateStoreCache.entrySet()) {
if (!columnNames.contains(entry.getKey())) {
continue;
}
ColumnHandle columnHandle = new ColumnHandle() {
@Override
public String getColumnName() {
return entry.getKey();
}
};
BloomFilterDynamicFilter newBloomDynamicFilter = new BloomFilterDynamicFilter("", columnHandle, entry.getValue(), GLOBAL);
for (Map<ColumnHandle, DynamicFilter> result : resultList) {
if (result.keySet().contains(entry.getKey())) {
DynamicFilter existsFilter = result.get(entry.getKey());
if (existsFilter instanceof BloomFilterDynamicFilter) {
BloomFilter existsBloomFilter = ((BloomFilterDynamicFilter) existsFilter).getBloomFilterDeserialized();
existsBloomFilter.merge(newBloomDynamicFilter.getBloomFilterDeserialized());
DynamicFilter newDynamicFilter = new BloomFilterDynamicFilter(existsFilter.getFilterId(), columnHandle, existsBloomFilter, GLOBAL);
result.put(columnHandle, newDynamicFilter);
}
} else {
result.put(columnHandle, newBloomDynamicFilter);
}
}
}
}
if (resultList.size() != context.getDisjunctSize()) {
return ImmutableList.of();
}
return resultList;
}
use of io.prestosql.spi.dynamicfilter.DynamicFilter in project hetu-core by openlookeng.
the class DataCenterPageSource method applyDynamicFilters.
private void applyDynamicFilters(Map<ColumnHandle, DynamicFilter> dynamicFilters) {
ImmutableMap.Builder<String, byte[]> builder = new ImmutableMap.Builder();
for (Map.Entry<ColumnHandle, DynamicFilter> entry : dynamicFilters.entrySet()) {
if (!appliedDynamicFilters.contains(entry.getKey().getColumnName())) {
DynamicFilter df = entry.getValue();
String columnName = entry.getKey().getColumnName();
if (df instanceof HashSetDynamicFilter) {
// FIXME: Read fpp from config
BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromHashSetDynamicFilter((HashSetDynamicFilter) df);
builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
} else if (df instanceof CombinedDynamicFilter) {
BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromCombinedDynamicFilter((CombinedDynamicFilter) df);
if (bloomFilterDynamicFilter != null) {
builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
}
} else if (df instanceof BloomFilterDynamicFilter) {
builder.put(columnName, ((BloomFilterDynamicFilter) df).getBloomFilterSerialized());
} else {
LOGGER.info("Dynamic Filter (type: " + df.getClass().getSimpleName() + ") skipped for DC connector");
}
}
}
Map<String, byte[]> newDynamicFilters = builder.build();
if (!newDynamicFilters.isEmpty()) {
if (client.applyDynamicFilters(newDynamicFilters)) {
appliedDynamicFilters.addAll(newDynamicFilters.keySet());
}
}
}
use of io.prestosql.spi.dynamicfilter.DynamicFilter in project hetu-core by openlookeng.
the class TestDynamicFilterCacheManager method testCacheAndRemoveDynamicFilters.
@Test
public void testCacheAndRemoveDynamicFilters() {
DynamicFilterCacheManager cacheManager = new DynamicFilterCacheManager();
String filterKey1 = createCacheKey("filter1", "query");
DynamicFilter filter1 = mock(DynamicFilter.class);
TaskId taskId = new TaskId("task1");
cacheManager.registerTask(filterKey1, taskId);
cacheManager.cacheDynamicFilter(filterKey1, filter1);
assertEquals(cacheManager.getDynamicFilter(filterKey1), filter1, "filter1 should be cached");
cacheManager.removeDynamicFilter(filterKey1, taskId);
assertNull(cacheManager.getDynamicFilter(filterKey1), "filter1 should be removed");
}
Aggregations