use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptCluster in project flink by apache.
the class FlinkRelDecorrelator method decorrelateQuery.
//~ Methods ----------------------------------------------------------------
/**
* Decorrelates a query.
* <p>
* <p>This is the main entry point to {@code FlinkRelDecorrelator}.
*
* @param rootRel Root node of the query
* @return Equivalent query with all
* {@link LogicalCorrelate} instances removed
*/
public static RelNode decorrelateQuery(RelNode rootRel) {
final CorelMap corelMap = new CorelMapBuilder().build(rootRel);
if (!corelMap.hasCorrelation()) {
return rootRel;
}
final RelOptCluster cluster = rootRel.getCluster();
final FlinkRelDecorrelator decorrelator = new FlinkRelDecorrelator(cluster, corelMap, cluster.getPlanner().getContext());
RelNode newRootRel = decorrelator.removeCorrelationViaRule(rootRel);
if (SQL2REL_LOGGER.isDebugEnabled()) {
SQL2REL_LOGGER.debug(RelOptUtil.dumpPlan("Plan after removing Correlator", newRootRel, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
if (!decorrelator.cm.mapCorVarToCorRel.isEmpty()) {
newRootRel = decorrelator.decorrelate(newRootRel);
}
return newRootRel;
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptCluster in project storm by apache.
the class TridentModifyRule method convert.
@Override
public RelNode convert(RelNode rel) {
final TableModify tableModify = (TableModify) rel;
final RelNode input = tableModify.getInput();
final RelOptCluster cluster = tableModify.getCluster();
final RelTraitSet traitSet = tableModify.getTraitSet().replace(TridentLogicalConvention.INSTANCE);
final RelOptTable relOptTable = tableModify.getTable();
final Prepare.CatalogReader catalogReader = tableModify.getCatalogReader();
final RelNode convertedInput = convert(input, input.getTraitSet().replace(TridentLogicalConvention.INSTANCE));
final TableModify.Operation operation = tableModify.getOperation();
final List<String> updateColumnList = tableModify.getUpdateColumnList();
final List<RexNode> sourceExpressionList = tableModify.getSourceExpressionList();
final boolean flattened = tableModify.isFlattened();
final Table table = tableModify.getTable().unwrap(Table.class);
switch(table.getJdbcTableType()) {
case STREAM:
if (operation != TableModify.Operation.INSERT) {
throw new UnsupportedOperationException(String.format("Streams doesn't support %s modify operation", operation));
}
return new TridentStreamInsertRel(cluster, traitSet, relOptTable, catalogReader, convertedInput, operation, updateColumnList, sourceExpressionList, flattened);
default:
throw new IllegalArgumentException(String.format("Unsupported table type: %s", table.getJdbcTableType()));
}
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptCluster in project hive by apache.
the class HiveSemiJoinRule method perform.
protected void perform(RelOptRuleCall call, ImmutableBitSet topRefs, RelNode topOperator, Join join, RelNode left, Aggregate aggregate) {
LOG.debug("Matched HiveSemiJoinRule");
final RelOptCluster cluster = join.getCluster();
final RexBuilder rexBuilder = cluster.getRexBuilder();
final ImmutableBitSet rightBits = ImmutableBitSet.range(left.getRowType().getFieldCount(), join.getRowType().getFieldCount());
if (topRefs.intersects(rightBits)) {
return;
}
final JoinInfo joinInfo = join.analyzeCondition();
if (!joinInfo.rightSet().equals(ImmutableBitSet.range(aggregate.getGroupCount()))) {
// By the way, neither a super-set nor a sub-set would work.
return;
}
if (join.getJoinType() == JoinRelType.LEFT) {
// since for LEFT join we are only interested in rows from LEFT we can get rid of right side
call.transformTo(topOperator.copy(topOperator.getTraitSet(), ImmutableList.of(left)));
return;
}
if (join.getJoinType() != JoinRelType.INNER) {
return;
}
if (!joinInfo.isEqui()) {
return;
}
LOG.debug("All conditions matched for HiveSemiJoinRule. Going to apply transformation.");
final List<Integer> newRightKeyBuilder = Lists.newArrayList();
final List<Integer> aggregateKeys = aggregate.getGroupSet().asList();
for (int key : joinInfo.rightKeys) {
newRightKeyBuilder.add(aggregateKeys.get(key));
}
final ImmutableIntList newRightKeys = ImmutableIntList.copyOf(newRightKeyBuilder);
final RelNode newRight = aggregate.getInput();
final RexNode newCondition = RelOptUtil.createEquiJoinCondition(left, joinInfo.leftKeys, newRight, newRightKeys, rexBuilder);
RelNode semi = null;
// is not expected further down the pipeline. see jira for more details
if (aggregate.getInput() instanceof HepRelVertex && ((HepRelVertex) aggregate.getInput()).getCurrentRel() instanceof Join) {
Join rightJoin = (Join) (((HepRelVertex) aggregate.getInput()).getCurrentRel());
List<RexNode> projects = new ArrayList<>();
for (int i = 0; i < rightJoin.getRowType().getFieldCount(); i++) {
projects.add(rexBuilder.makeInputRef(rightJoin, i));
}
RelNode topProject = call.builder().push(rightJoin).project(projects, rightJoin.getRowType().getFieldNames(), true).build();
semi = call.builder().push(left).push(topProject).semiJoin(newCondition).build();
} else {
semi = call.builder().push(left).push(aggregate.getInput()).semiJoin(newCondition).build();
}
call.transformTo(topOperator.copy(topOperator.getTraitSet(), ImmutableList.of(semi)));
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptCluster in project hive by apache.
the class HiveExceptRewriteRule method onMatch.
// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
final HiveExcept hiveExcept = call.rel(0);
final RelOptCluster cluster = hiveExcept.getCluster();
final RexBuilder rexBuilder = cluster.getRexBuilder();
Builder<RelNode> bldr = new ImmutableList.Builder<RelNode>();
// branch
try {
bldr.add(createFirstGB(hiveExcept.getInputs().get(0), true, cluster, rexBuilder));
bldr.add(createFirstGB(hiveExcept.getInputs().get(1), false, cluster, rexBuilder));
} catch (CalciteSemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
// create a union above all the branches
// the schema of union looks like this
// all keys + VCol + c
HiveRelNode union = new HiveUnion(cluster, TraitsUtil.getDefaultTraitSet(cluster), bldr.build());
// 2nd level GB: create a GB (all keys + sum(c) as a + sum(VCol*c) as b) for
// each branch
final List<RexNode> gbChildProjLst = Lists.newArrayList();
final List<Integer> groupSetPositions = Lists.newArrayList();
int unionColumnSize = union.getRowType().getFieldList().size();
for (int cInd = 0; cInd < unionColumnSize; cInd++) {
gbChildProjLst.add(rexBuilder.makeInputRef(union, cInd));
// the last 2 columns are VCol and c
if (cInd < unionColumnSize - 2) {
groupSetPositions.add(cInd);
}
}
try {
gbChildProjLst.add(multiply(rexBuilder.makeInputRef(union, unionColumnSize - 2), rexBuilder.makeInputRef(union, unionColumnSize - 1), cluster, rexBuilder));
} catch (CalciteSemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
RelNode gbInputRel = null;
try {
// Here we create a project for the following reasons:
// (1) GBy only accepts arg as a position of the input, however, we need to sum on VCol*c
// (2) This can better reuse the function createSingleArgAggCall.
gbInputRel = HiveProject.create(union, gbChildProjLst, null);
} catch (CalciteSemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
// gbInputRel's schema is like this
// all keys + VCol + c + VCol*c
List<AggregateCall> aggregateCalls = Lists.newArrayList();
RelDataType aggFnRetType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory());
// sum(c)
AggregateCall aggregateCall = HiveCalciteUtil.createSingleArgAggCall("sum", cluster, TypeInfoFactory.longTypeInfo, unionColumnSize - 1, aggFnRetType);
aggregateCalls.add(aggregateCall);
// sum(VCol*c)
aggregateCall = HiveCalciteUtil.createSingleArgAggCall("sum", cluster, TypeInfoFactory.longTypeInfo, unionColumnSize, aggFnRetType);
aggregateCalls.add(aggregateCall);
final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions);
HiveRelNode aggregateRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, groupSet, null, aggregateCalls);
if (!hiveExcept.all) {
RelNode filterRel = null;
try {
filterRel = new HiveFilter(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), aggregateRel, makeFilterExprForExceptDistinct(aggregateRel, unionColumnSize, cluster, rexBuilder));
} catch (CalciteSemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
// finally add a project to project out the last 2 columns
Set<Integer> projectOutColumnPositions = new HashSet<>();
projectOutColumnPositions.add(filterRel.getRowType().getFieldList().size() - 2);
projectOutColumnPositions.add(filterRel.getRowType().getFieldList().size() - 1);
try {
call.transformTo(HiveCalciteUtil.createProjectWithoutColumn(filterRel, projectOutColumnPositions));
} catch (CalciteSemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
} else {
List<RexNode> originalInputRefs = Lists.transform(aggregateRel.getRowType().getFieldList(), new Function<RelDataTypeField, RexNode>() {
@Override
public RexNode apply(RelDataTypeField input) {
return new RexInputRef(input.getIndex(), input.getType());
}
});
List<RexNode> copyInputRefs = new ArrayList<>();
try {
copyInputRefs.add(makeExprForExceptAll(aggregateRel, unionColumnSize, cluster, rexBuilder));
} catch (CalciteSemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
for (int i = 0; i < originalInputRefs.size() - 2; i++) {
copyInputRefs.add(originalInputRefs.get(i));
}
RelNode srcRel = null;
try {
srcRel = HiveProject.create(aggregateRel, copyInputRefs, null);
HiveTableFunctionScan udtf = HiveCalciteUtil.createUDTFForSetOp(cluster, srcRel);
// finally add a project to project out the 1st columns
Set<Integer> projectOutColumnPositions = new HashSet<>();
projectOutColumnPositions.add(0);
call.transformTo(HiveCalciteUtil.createProjectWithoutColumn(udtf, projectOutColumnPositions));
} catch (SemanticException e) {
LOG.debug(e.toString());
throw new RuntimeException(e);
}
}
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptCluster in project hive by apache.
the class TestHivePointLookupOptimizerRule method before.
@Before
public void before() {
HepProgramBuilder programBuilder = new HepProgramBuilder();
programBuilder.addRuleInstance(new HivePointLookupOptimizerRule.FilterCondition(2));
planner = new HepPlanner(programBuilder.build());
JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl();
RexBuilder rexBuilder = new RexBuilder(typeFactory);
final RelOptCluster optCluster = RelOptCluster.create(planner, rexBuilder);
RelDataType rowTypeMock = typeFactory.createStructType(MyRecord.class);
doReturn(rowTypeMock).when(tableMock).getRowType();
LogicalTableScan tableScan = LogicalTableScan.create(optCluster, tableMock, Collections.emptyList());
doReturn(tableScan).when(tableMock).toRel(ArgumentMatchers.any());
doReturn(tableMock).when(schemaMock).getTableForMember(any());
lenient().doReturn(hiveTableMDMock).when(tableMock).getHiveTableMD();
builder = HiveRelFactories.HIVE_BUILDER.create(optCluster, schemaMock);
}
Aggregations