Examples with RelDataTypeFactory org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory used on opensource projects

Example 91 with RelDataTypeFactory

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory in project hive by apache.

the class HiveFilterProjectTSTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    final Filter filter = call.rel(0);
    final Project project = call.rel(1);
    if (RexOver.containsOver(project.getProjects(), null)) {
        // it can be pushed down. For now we don't support this.
        return;
    }
    if (RexUtil.containsCorrelation(filter.getCondition())) {
        // Correlate from being de-correlated.
        return;
    }
    // convert the filter to one that references the child of the project
    RexNode newCondition = RelOptUtil.pushPastProject(filter.getCondition(), project);
    // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
    // nullable and not-nullable conditions, but a CAST might get in the way of
    // other rewrites.
    final RelDataTypeFactory typeFactory = filter.getCluster().getTypeFactory();
    if (RexUtil.isNullabilityCast(typeFactory, newCondition)) {
        newCondition = ((RexCall) newCondition).getOperands().get(0);
    }
    RelNode newFilterRel = filterFactory == null ? filter.copy(filter.getTraitSet(), project.getInput(), newCondition) : filterFactory.createFilter(project.getInput(), newCondition);
    RelNode newProjRel = projectFactory == null ? project.copy(project.getTraitSet(), newFilterRel, project.getProjects(), project.getRowType()) : projectFactory.createProject(newFilterRel, Collections.emptyList(), project.getProjects(), project.getRowType().getFieldNames());
    call.transformTo(newProjRel);
}

Example 92 with RelDataTypeFactory

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory in project hive by apache.

the class HiveMaterializedViewsRegistry method createMaterializedViewScan.

private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable) {
    // 0. Recreate cluster
    final RelOptPlanner planner = CalcitePlanner.createPlanner(conf);
    final RexBuilder rexBuilder = new RexBuilder(new JavaTypeFactoryImpl(new HiveTypeSystemImpl()));
    final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
    // 1. Create column schema
    final RowResolver rr = new RowResolver();
    // 1.1 Add Column info for non partion cols (Object Inspector fields)
    StructObjectInspector rowObjectInspector;
    try {
        rowObjectInspector = (StructObjectInspector) viewTable.getDeserializer().getObjectInspector();
    } catch (SerDeException e) {
        // Bail out
        return null;
    }
    List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
    ColumnInfo colInfo;
    String colName;
    ArrayList<ColumnInfo> cInfoLst = new ArrayList<>();
    for (StructField structField : fields) {
        colName = structField.getFieldName();
        colInfo = new ColumnInfo(structField.getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(structField.getFieldObjectInspector()), null, false);
        rr.put(null, colName, colInfo);
        cInfoLst.add(colInfo);
    }
    ArrayList<ColumnInfo> nonPartitionColumns = new ArrayList<ColumnInfo>(cInfoLst);
    // 1.2 Add column info corresponding to partition columns
    ArrayList<ColumnInfo> partitionColumns = new ArrayList<ColumnInfo>();
    for (FieldSchema part_col : viewTable.getPartCols()) {
        colName = part_col.getName();
        colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true);
        rr.put(null, colName, colInfo);
        cInfoLst.add(colInfo);
        partitionColumns.add(colInfo);
    }
    // 1.3 Build row type from field <type, name>
    RelDataType rowType;
    try {
        rowType = TypeConverter.getType(cluster, rr, null);
    } catch (CalciteSemanticException e) {
        // Bail out
        return null;
    }
    // 2. Build RelOptAbstractTable
    List<String> fullyQualifiedTabName = new ArrayList<>();
    if (viewTable.getDbName() != null && !viewTable.getDbName().isEmpty()) {
        fullyQualifiedTabName.add(viewTable.getDbName());
    }
    fullyQualifiedTabName.add(viewTable.getTableName());
    RelNode tableRel;
    // 3. Build operator
    if (obtainTableType(viewTable) == TableType.DRUID) {
        // Build Druid query
        String address = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
        String dataSource = viewTable.getParameters().get(Constants.DRUID_DATA_SOURCE);
        Set<String> metrics = new HashSet<>();
        List<RelDataType> druidColTypes = new ArrayList<>();
        List<String> druidColNames = new ArrayList<>();
        // @NOTE this code is very similar to the code at org/apache/hadoop/hive/ql/parse/CalcitePlanner.java:2362
        // @TODO it will be nice to refactor it
        RelDataTypeFactory dtFactory = cluster.getRexBuilder().getTypeFactory();
        for (RelDataTypeField field : rowType.getFieldList()) {
            if (DruidTable.DEFAULT_TIMESTAMP_COLUMN.equals(field.getName())) {
                // Druid's time column is always not null.
                druidColTypes.add(dtFactory.createTypeWithNullability(field.getType(), false));
            } else {
                druidColTypes.add(field.getType());
            }
            druidColNames.add(field.getName());
            if (field.getName().equals(DruidTable.DEFAULT_TIMESTAMP_COLUMN)) {
                // timestamp
                continue;
            }
            if (field.getType().getSqlTypeName() == SqlTypeName.VARCHAR) {
                // dimension
                continue;
            }
            metrics.add(field.getName());
        }
        List<Interval> intervals = Collections.singletonList(DruidTable.DEFAULT_INTERVAL);
        rowType = dtFactory.createStructType(druidColTypes, druidColNames);
        // We can pass null for Hive object because it is only used to retrieve tables
        // if constraints on a table object are existing, but constraints cannot be defined
        // for materialized views.
        RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), conf, null, new QueryTables(true), new HashMap<>(), new HashMap<>(), new AtomicInteger());
        DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals, null, null);
        final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false);
        tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), optTable, druidTable, ImmutableList.<RelNode>of(scan), ImmutableMap.of());
    } else {
        // Build Hive Table Scan Rel.
        // We can pass null for Hive object because it is only used to retrieve tables
        // if constraints on a table object are existing, but constraints cannot be defined
        // for materialized views.
        RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), conf, null, new QueryTables(true), new HashMap<>(), new HashMap<>(), new AtomicInteger());
        tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false);
    }
    return tableRel;
}

Example 93 with RelDataTypeFactory

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory in project hive by apache.

the class HiveAggregateReduceFunctionsRule method reduceAvg.

private RexNode reduceAvg(Aggregate oldAggRel, AggregateCall oldCall, List<AggregateCall> newCalls, Map<AggregateCall, RexNode> aggCallMapping, List<RexNode> inputExprs) {
    final int nGroups = oldAggRel.getGroupCount();
    final RexBuilder rexBuilder = oldAggRel.getCluster().getRexBuilder();
    final RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory();
    final int iAvgInput = oldCall.getArgList().get(0);
    final RelDataType avgInputType = typeFactory.createTypeWithNullability(getFieldType(oldAggRel.getInput(), iAvgInput), true);
    final RelDataType sumReturnType = getSumReturnType(rexBuilder.getTypeFactory(), avgInputType);
    final AggregateCall sumCall = AggregateCall.create(new HiveSqlSumAggFunction(oldCall.isDistinct(), ReturnTypes.explicit(sumReturnType), oldCall.getAggregation().getOperandTypeInference(), // SqlStdOperatorTable.SUM,
    oldCall.getAggregation().getOperandTypeChecker()), oldCall.isDistinct(), oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), oldAggRel.getInput(), null, null);
    RelDataType countRetType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true);
    final AggregateCall countCall = AggregateCall.create(new HiveSqlCountAggFunction(oldCall.isDistinct(), ReturnTypes.explicit(countRetType), oldCall.getAggregation().getOperandTypeInference(), // SqlStdOperatorTable.COUNT,
    oldCall.getAggregation().getOperandTypeChecker()), oldCall.isDistinct(), oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), oldAggRel.getInput(), countRetType, null);
    // NOTE:  these references are with respect to the output
    // of newAggRel
    RexNode numeratorRef = rexBuilder.addAggCall(sumCall, nGroups, oldAggRel.indicator, newCalls, aggCallMapping, ImmutableList.of(avgInputType));
    final RexNode denominatorRef = rexBuilder.addAggCall(countCall, nGroups, oldAggRel.indicator, newCalls, aggCallMapping, ImmutableList.of(avgInputType));
    if (numeratorRef.getType().getSqlTypeName() != SqlTypeName.DECIMAL) {
        // If type is not decimal, we enforce the same type as the avg to comply with
        // Hive semantics
        numeratorRef = rexBuilder.ensureType(oldCall.getType(), numeratorRef, true);
    }
    final RexNode divideRef = rexBuilder.makeCall(SqlStdOperatorTable.DIVIDE, numeratorRef, denominatorRef);
    return rexBuilder.makeCast(oldCall.getType(), divideRef);
}

Example 94 with RelDataTypeFactory

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory in project hive by apache.

the class HiveAggregateReduceFunctionsRule method reduceSum0.

private RexNode reduceSum0(Aggregate oldAggRel, AggregateCall oldCall, List<AggregateCall> newCalls, Map<AggregateCall, RexNode> aggCallMapping, List<RexNode> inputExprs) {
    final int nGroups = oldAggRel.getGroupCount();
    final RexBuilder rexBuilder = oldAggRel.getCluster().getRexBuilder();
    final RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory();
    final int iAvgInput = oldCall.getArgList().get(0);
    final RelDataType sum0InputType = typeFactory.createTypeWithNullability(getFieldType(oldAggRel.getInput(), iAvgInput), true);
    final RelDataType sumReturnType = getSumReturnType(rexBuilder.getTypeFactory(), sum0InputType);
    final AggregateCall sumCall = AggregateCall.create(new HiveSqlSumAggFunction(oldCall.isDistinct(), ReturnTypes.explicit(sumReturnType), oldCall.getAggregation().getOperandTypeInference(), // SqlStdOperatorTable.SUM,
    oldCall.getAggregation().getOperandTypeChecker()), oldCall.isDistinct(), oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), oldAggRel.getInput(), null, null);
    RexNode refSum = rexBuilder.addAggCall(sumCall, nGroups, oldAggRel.indicator, newCalls, aggCallMapping, ImmutableList.of(sum0InputType));
    refSum = rexBuilder.ensureType(oldCall.getType(), refSum, true);
    final RexNode coalesce = rexBuilder.makeCall(SqlStdOperatorTable.COALESCE, refSum, rexBuilder.makeZeroLiteral(refSum.getType()));
    return rexBuilder.makeCast(oldCall.getType(), coalesce);
}

Example 95 with RelDataTypeFactory

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory in project hive by apache.

the class HiveAggregate method deriveRowType.

public static RelDataType deriveRowType(RelDataTypeFactory typeFactory, final RelDataType inputRowType, boolean indicator, ImmutableBitSet groupSet, final List<AggregateCall> aggCalls) {
    final List<Integer> groupList = groupSet.asList();
    assert groupList.size() == groupSet.cardinality();
    final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
    final List<RelDataTypeField> fieldList = inputRowType.getFieldList();
    final Set<String> containedNames = Sets.newHashSet();
    for (int groupKey : groupList) {
        containedNames.add(fieldList.get(groupKey).getName());
        builder.add(fieldList.get(groupKey));
    }
    if (indicator) {
        for (int groupKey : groupList) {
            final RelDataType booleanType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BOOLEAN), false);
            String name = "i$" + fieldList.get(groupKey).getName();
            int i = 0;
            StringBuilder nameBuilder = new StringBuilder(name);
            while (containedNames.contains(name)) {
                nameBuilder.append('_').append(i++);
            }
            containedNames.add(nameBuilder.toString());
            builder.add(name, booleanType);
        }
    }
    for (Ord<AggregateCall> aggCall : Ord.zip(aggCalls)) {
        String name;
        if (aggCall.e.name != null) {
            name = aggCall.e.name;
        } else {
            name = "$f" + (groupList.size() + aggCall.i);
        }
        int i = 0;
        while (containedNames.contains(name)) {
            name += "_" + i++;
        }
        containedNames.add(name);
        builder.add(name, aggCall.e.type);
    }
    return builder.build();
}