Search in sources :

Example 1 with HiveOffloadExpression

use of io.prestosql.plugin.hive.HiveOffloadExpression in project boostkit-bigdata by kunpengcompute.

the class PageSourceUtil method buildColumnsProjections.

public static List<RowExpression> buildColumnsProjections(HiveOffloadExpression expression, List<HiveColumnHandle> columns, Map<VariableReferenceExpression, Integer> layoutMap, Map<VariableReferenceExpression, Integer> projectionsLayout) {
    checkArgument(projectionsLayout.isEmpty(), "buildColumnsProjections: input reference error.");
    int channel = 0;
    ImmutableList.Builder<RowExpression> builder = ImmutableList.builder();
    if (!expression.getProjections().isEmpty()) {
        for (Map.Entry<Symbol, RowExpression> entry : expression.getProjections().entrySet()) {
            RowExpression rowExpression = VariableToChannelTranslator.translate(entry.getValue(), layoutMap);
            builder.add(rowExpression);
            projectionsLayout.put(new VariableReferenceExpression(entry.getKey().getName(), rowExpression.getType()), channel++);
        }
        return builder.build();
    }
    Map<String, Integer> nameMap = layoutMap.entrySet().stream().collect(toMap(key -> key.getKey().getName(), val -> val.getValue()));
    Map<String, Type> typeMap = layoutMap.entrySet().stream().collect(toMap(key -> key.getKey().getName(), val -> val.getKey().getType()));
    Set<String> columnSet = new HashSet<>();
    for (HiveColumnHandle columnHandle : columns) {
        if (columnHandle.getHiveColumnIndex() == DUMMY_OFFLOADED_COLUMN_INDEX) {
            continue;
        }
        if (columnSet.add(columnHandle.getName())) {
            Type type = typeMap.get(columnHandle.getName());
            InputReferenceExpression inputReferenceExpression = new InputReferenceExpression(nameMap.get(columnHandle.getName()), type);
            projectionsLayout.put(new VariableReferenceExpression(columnHandle.getName(), type), channel++);
            builder.add(inputReferenceExpression);
        }
    }
    return builder.build();
}
Also used : TRUE_CONSTANT(io.prestosql.expressions.LogicalRowExpressions.TRUE_CONSTANT) HivePartitionKey(io.prestosql.plugin.hive.HivePartitionKey) HashMap(java.util.HashMap) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle) OptionalInt(java.util.OptionalInt) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) CallExpression(io.prestosql.spi.relation.CallExpression) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toMap(java.util.stream.Collectors.toMap) PARTITION_KEY(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Path(org.apache.hadoop.fs.Path) Predicate(com.huawei.boostkit.omnidata.model.Predicate) Column(com.huawei.boostkit.omnidata.model.Column) Type(io.prestosql.spi.type.Type) Symbol(io.prestosql.spi.plan.Symbol) DUMMY_OFFLOADED_COLUMN_INDEX(io.prestosql.plugin.hive.HiveColumnHandle.DUMMY_OFFLOADED_COLUMN_INDEX) ImmutableMap(com.google.common.collect.ImmutableMap) HiveUtil(io.prestosql.plugin.hive.HiveUtil) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TypeManager(io.prestosql.spi.type.TypeManager) Set(java.util.Set) IOException(java.io.IOException) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) VariableToChannelTranslator(io.prestosql.spi.relation.VariableToChannelTranslator) Collectors(java.util.stream.Collectors) AggregationInfo(com.huawei.boostkit.omnidata.model.AggregationInfo) List(java.util.List) HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) DUMMY_OFFLOADED(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.DUMMY_OFFLOADED) Domain(io.prestosql.spi.predicate.Domain) InputReferenceExpression(io.prestosql.spi.relation.InputReferenceExpression) RowExpression(io.prestosql.spi.relation.RowExpression) Optional(java.util.Optional) InputReferenceExpression(io.prestosql.spi.relation.InputReferenceExpression) ImmutableList(com.google.common.collect.ImmutableList) Symbol(io.prestosql.spi.plan.Symbol) RowExpression(io.prestosql.spi.relation.RowExpression) Type(io.prestosql.spi.type.Type) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) HashMap(java.util.HashMap) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle) HashSet(java.util.HashSet)

Example 2 with HiveOffloadExpression

use of io.prestosql.plugin.hive.HiveOffloadExpression in project boostkit-bigdata by kunpengcompute.

the class HiveProjectPushdown method tryProjectPushdown.

protected static Optional<TableScanNode> tryProjectPushdown(ProjectNode plan, Map<String, Type> types) {
    if (!(plan.getSource() instanceof TableScanNode)) {
        return Optional.empty();
    }
    TableScanNode tableScanNode = (TableScanNode) plan.getSource();
    ConnectorTableHandle tableHandle = tableScanNode.getTable().getConnectorHandle();
    if (!(tableHandle instanceof HiveTableHandle) || !(((HiveTableHandle) tableHandle).getOffloadExpression().getProjections().isEmpty())) {
        return Optional.empty();
    }
    Map<Symbol, ColumnHandle> assignments = new HashMap<>();
    HiveTypeTranslator hiveTypeTranslator = new HiveTypeTranslator();
    for (Map.Entry<Symbol, RowExpression> entry : plan.getAssignments().entrySet()) {
        String name = entry.getKey().getName();
        HiveType hiveType = HiveType.toHiveType(hiveTypeTranslator, entry.getValue().getType());
        TypeSignature typeSignature = entry.getValue().getType().getTypeSignature();
        HiveColumnHandle columnHandle = new HiveColumnHandle(name, hiveType, typeSignature, DUMMY_OFFLOADED_COLUMN_INDEX, DUMMY_OFFLOADED, Optional.of("projections pushed down " + name));
        assignments.put(entry.getKey(), columnHandle);
    }
    BiMap<VariableReferenceExpression, VariableReferenceExpression> variableToColumnMapping = tableScanNode.getAssignments().entrySet().stream().collect(toImmutableBiMap(entry -> new VariableReferenceExpression(entry.getKey().getName(), types.get(entry.getKey().getName())), entry -> new VariableReferenceExpression(entry.getValue().getColumnName(), types.get(entry.getKey().getName()))));
    ImmutableMap.Builder<Symbol, RowExpression> projections = new ImmutableMap.Builder<>();
    for (Map.Entry<Symbol, RowExpression> entry : plan.getAssignments().getMap().entrySet()) {
        RowExpression expression = replaceExpression(entry.getValue(), variableToColumnMapping);
        if (!OmniExpressionChecker.checkExpression(expression)) {
            return Optional.empty();
        }
        projections.put(entry.getKey(), expression);
    }
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    HiveOffloadExpression offloadExpression = hiveTableHandle.getOffloadExpression().updateProjections(projections.build(), getDataSourceColumns(tableScanNode));
    HiveTableHandle newHiveTableHandle = hiveTableHandle.withOffloadExpression(offloadExpression);
    TableHandle newTableHandle = new TableHandle(tableScanNode.getTable().getCatalogName(), newHiveTableHandle, tableScanNode.getTable().getTransaction(), tableScanNode.getTable().getLayout());
    return Optional.of(new TableScanNode(tableScanNode.getId(), newTableHandle, ImmutableList.copyOf(assignments.keySet()), assignments, tableScanNode.getEnforcedConstraint(), tableScanNode.getPredicate(), tableScanNode.getStrategy(), tableScanNode.getReuseTableScanMappingId(), tableScanNode.getConsumerTableScanNodeCount(), tableScanNode.isForDelete()));
}
Also used : HiveTableHandle(io.prestosql.plugin.hive.HiveTableHandle) HivePushdownUtil.getDataSourceColumns(io.prestosql.plugin.hive.rule.HivePushdownUtil.getDataSourceColumns) HiveTypeTranslator(io.prestosql.plugin.hive.HiveTypeTranslator) HashMap(java.util.HashMap) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) ImmutableList(com.google.common.collect.ImmutableList) OmniExpressionChecker(com.huawei.boostkit.omnidata.expression.OmniExpressionChecker) Map(java.util.Map) Type(io.prestosql.spi.type.Type) RowExpressionNodeInliner.replaceExpression(io.prestosql.expressions.RowExpressionNodeInliner.replaceExpression) Symbol(io.prestosql.spi.plan.Symbol) BiMap(com.google.common.collect.BiMap) DUMMY_OFFLOADED_COLUMN_INDEX(io.prestosql.plugin.hive.HiveColumnHandle.DUMMY_OFFLOADED_COLUMN_INDEX) ImmutableMap(com.google.common.collect.ImmutableMap) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) HiveType(io.prestosql.plugin.hive.HiveType) ProjectNode(io.prestosql.spi.plan.ProjectNode) HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) DUMMY_OFFLOADED(io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.DUMMY_OFFLOADED) RowExpression(io.prestosql.spi.relation.RowExpression) Optional(java.util.Optional) ImmutableBiMap.toImmutableBiMap(com.google.common.collect.ImmutableBiMap.toImmutableBiMap) TypeSignature(io.prestosql.spi.type.TypeSignature) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) RowExpression(io.prestosql.spi.relation.RowExpression) ImmutableMap(com.google.common.collect.ImmutableMap) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) TypeSignature(io.prestosql.spi.type.TypeSignature) HiveTableHandle(io.prestosql.plugin.hive.HiveTableHandle) HiveTypeTranslator(io.prestosql.plugin.hive.HiveTypeTranslator) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression) HiveTableHandle(io.prestosql.plugin.hive.HiveTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) HiveType(io.prestosql.plugin.hive.HiveType) HashMap(java.util.HashMap) Map(java.util.Map) BiMap(com.google.common.collect.BiMap) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableBiMap.toImmutableBiMap(com.google.common.collect.ImmutableBiMap.toImmutableBiMap) HiveColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle)

Example 3 with HiveOffloadExpression

use of io.prestosql.plugin.hive.HiveOffloadExpression in project boostkit-bigdata by kunpengcompute.

the class TestHivePushdownUtil method matchFilterOffload.

protected static void matchFilterOffload(PlanNode node, RowExpression predicate) {
    HiveOffloadExpression expression = getCheckedOffloadExpression(node);
    assertTrue(expression.isPresent());
    assertEquals(predicate, expression.getFilterExpression());
}
Also used : HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression)

Example 4 with HiveOffloadExpression

use of io.prestosql.plugin.hive.HiveOffloadExpression in project boostkit-bigdata by kunpengcompute.

the class TestHivePushdownUtil method matchLimitOffload.

protected static void matchLimitOffload(PlanNode node, long count) {
    HiveOffloadExpression expression = getCheckedOffloadExpression(node);
    assertTrue(expression.isPresent());
    assertEquals(count, expression.getLimit().getAsLong());
}
Also used : HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression)

Example 5 with HiveOffloadExpression

use of io.prestosql.plugin.hive.HiveOffloadExpression in project boostkit-bigdata by kunpengcompute.

the class TestHivePushdownUtil method getCheckedOffloadExpression.

private static HiveOffloadExpression getCheckedOffloadExpression(PlanNode node) {
    assertTrue(node instanceof TableScanNode);
    ConnectorTableHandle tableHandle = ((TableScanNode) node).getTable().getConnectorHandle();
    assertTrue(tableHandle instanceof HiveTableHandle);
    HiveOffloadExpression hiveOffloadExpression = ((HiveTableHandle) tableHandle).getOffloadExpression();
    return hiveOffloadExpression;
}
Also used : HiveTableHandle(io.prestosql.plugin.hive.HiveTableHandle) TableScanNode(io.prestosql.spi.plan.TableScanNode) HiveOffloadExpression(io.prestosql.plugin.hive.HiveOffloadExpression) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle)

Aggregations

HiveOffloadExpression (io.prestosql.plugin.hive.HiveOffloadExpression)9 ImmutableList (com.google.common.collect.ImmutableList)3 HiveColumnHandle (io.prestosql.plugin.hive.HiveColumnHandle)3 HiveTableHandle (io.prestosql.plugin.hive.HiveTableHandle)3 ConnectorTableHandle (io.prestosql.spi.connector.ConnectorTableHandle)3 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 BiMap (com.google.common.collect.BiMap)2 ImmutableBiMap.toImmutableBiMap (com.google.common.collect.ImmutableBiMap.toImmutableBiMap)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 OmniExpressionChecker (com.huawei.boostkit.omnidata.expression.OmniExpressionChecker)2 AggregationInfo (com.huawei.boostkit.omnidata.model.AggregationInfo)2 TRUE_CONSTANT (io.prestosql.expressions.LogicalRowExpressions.TRUE_CONSTANT)2 RowExpressionNodeInliner.replaceExpression (io.prestosql.expressions.RowExpressionNodeInliner.replaceExpression)2 DUMMY_OFFLOADED (io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.DUMMY_OFFLOADED)2 DUMMY_OFFLOADED_COLUMN_INDEX (io.prestosql.plugin.hive.HiveColumnHandle.DUMMY_OFFLOADED_COLUMN_INDEX)2 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)2 TableHandle (io.prestosql.spi.metadata.TableHandle)2 Symbol (io.prestosql.spi.plan.Symbol)2 TableScanNode (io.prestosql.spi.plan.TableScanNode)2 RowExpression (io.prestosql.spi.relation.RowExpression)2