use of io.prestosql.plugin.hive.HiveMetadata in project boostkit-bigdata by kunpengcompute.
the class HiveFilterPushdown method evaluateFilterBenefit.
private static boolean evaluateFilterBenefit(ConnectorTableHandle tableHandle, Map<String, ColumnHandle> columnHandlesMap, HiveMetadata metadata, FilterStatsCalculatorService filterCalculatorService, RowExpression predicate, Constraint constraint, ConnectorSession session, Map<String, Type> typesMap) {
TableStatistics statistics = metadata.getTableStatistics(session, tableHandle, constraint, true);
if (statistics.getRowCount().isUnknown() || statistics.getRowCount().getValue() < HiveSessionProperties.getMinOffloadRowNumber(session)) {
log.info("Filter:Table %s row number[%d], expect min row number[%d], predicate[%s].", tableHandle.getTableName(), (long) statistics.getRowCount().getValue(), HiveSessionProperties.getMinOffloadRowNumber(session), predicate.toString());
return false;
}
Set<String> predicateVariables = HivePushdownUtil.extractAll(predicate).stream().map(VariableReferenceExpression::getName).collect(Collectors.toSet());
Map<ColumnHandle, String> allColumns = columnHandlesMap.entrySet().stream().filter(entry -> predicateVariables.contains(entry.getKey())).collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
Map<String, Type> allColumnTypes = allColumns.entrySet().stream().collect(toImmutableMap(entry -> entry.getValue(), entry -> metadata.getColumnMetadata(session, tableHandle, entry.getKey()).getType()));
Map<Symbol, Type> symbolsMap = typesMap.entrySet().stream().collect(Collectors.toMap(entry -> new Symbol(entry.getKey()), entry -> entry.getValue()));
allColumnTypes.forEach((key, value) -> {
if (!symbolsMap.containsKey(key)) {
symbolsMap.put(new Symbol(key), value);
}
});
TableStatistics filterStatistics = filterCalculatorService.filterStats(statistics, predicate, session, allColumns, allColumnTypes, symbolsMap, formSymbolsLayout(allColumns));
Estimate filteredRowCount = filterStatistics.getRowCount().isUnknown() ? statistics.getRowCount() : filterStatistics.getRowCount();
double filterFactor = filteredRowCount.getValue() / statistics.getRowCount().getValue();
if (filterFactor <= HiveSessionProperties.getFilterOffloadFactor(session)) {
log.info("Offloading: table %s, size[%d], predicate[%s], filter factor[%.2f%%].", tableHandle.getTableName(), (long) statistics.getRowCount().getValue(), predicate.toString(), filterFactor * 100);
return true;
} else {
log.info("No need to offload: table %s, size[%d], predicate[%s], filter factor[%.2f%%].", tableHandle.getTableName(), (long) statistics.getRowCount().getValue(), predicate.toString(), filterFactor * 100);
}
return false;
}
use of io.prestosql.plugin.hive.HiveMetadata in project boostkit-bigdata by kunpengcompute.
the class HiveFilterPushdown method getMetadata.
private HiveMetadata getMetadata(TableHandle tableHandle) {
ConnectorMetadata metadata = transactionManager.get(tableHandle.getTransaction());
checkState(metadata instanceof HiveMetadata, "metadata must be HiveMetadata");
return (HiveMetadata) metadata;
}
use of io.prestosql.plugin.hive.HiveMetadata in project boostkit-bigdata by kunpengcompute.
the class TestHivePartialAggregationPushdown method createOptimizer.
private static HivePartialAggregationPushdown createOptimizer() {
HiveMetadataFactory hiveMetadataFactory = Mockito.mock(HiveMetadataFactory.class);
HiveMetadata hiveMetadata = simulationHiveMetadata();
Mockito.when(hiveMetadataFactory.get()).thenReturn(hiveMetadata);
StandardFunctionResolution resolution = new FunctionResolution(OFFLOAD_METADATA.getFunctionAndTypeManager());
HivePartialAggregationPushdown optimizer = new HivePartialAggregationPushdown(simulationHiveTransactionManager(), OFFLOAD_METADATA.getFunctionAndTypeManager(), resolution, hiveMetadataFactory);
return optimizer;
}
use of io.prestosql.plugin.hive.HiveMetadata in project boostkit-bigdata by kunpengcompute.
the class TestHivePlanOptimizerProvider method testProvider.
@Test
public void testProvider() {
RowExpressionService expressionService = new ConnectorRowExpressionService(new RowExpressionDomainTranslator(OFFLOAD_METADATA), new RowExpressionDeterminismEvaluator(OFFLOAD_METADATA));
HiveTransactionManager transactionManager = simulationHiveTransactionManager();
StandardFunctionResolution resolution = new FunctionResolution(OFFLOAD_METADATA.getFunctionAndTypeManager());
HivePartitionManager partitionManager = new HivePartitionManager(OFFLOAD_METADATA.getFunctionAndTypeManager(), 1, false, 1);
ScalarStatsCalculator scalarStatsCalculator = new ScalarStatsCalculator(OFFLOAD_METADATA);
StatsNormalizer normalizer = new StatsNormalizer();
FilterStatsCalculator statsCalculator = new FilterStatsCalculator(OFFLOAD_METADATA, scalarStatsCalculator, normalizer);
FilterStatsCalculatorService calculatorService = new ConnectorFilterStatsCalculatorService(statsCalculator);
HiveMetadataFactory hiveMetadataFactory = Mockito.mock(HiveMetadataFactory.class);
HiveMetadata hiveMetadata = simulationHiveMetadata();
Mockito.when(hiveMetadataFactory.get()).thenReturn(hiveMetadata);
HivePlanOptimizerProvider hivePlanOptimizerProvider = new HivePlanOptimizerProvider(transactionManager, expressionService, resolution, partitionManager, OFFLOAD_METADATA.getFunctionAndTypeManager(), calculatorService, hiveMetadataFactory);
assertEquals(hivePlanOptimizerProvider.getLogicalPlanOptimizers().size(), 3);
assertEquals(hivePlanOptimizerProvider.getPhysicalPlanOptimizers().size(), 3);
}
use of io.prestosql.plugin.hive.HiveMetadata in project boostkit-bigdata by kunpengcompute.
the class TestHivePushdownUtil method simulationHiveTransactionManager.
protected static HiveTransactionManager simulationHiveTransactionManager() {
HiveMetadata metadata = simulationHiveMetadata();
HiveTransactionManager transactionManager = Mockito.mock(HiveTransactionManager.class);
Mockito.when(transactionManager.get(OFFLOAD_TABLE_HANDLE.getTransaction())).thenReturn(metadata);
return transactionManager;
}
Aggregations