use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral in project samza by apache.
the class JoinTranslator method validateJoinQuery.
private void validateJoinQuery(LogicalJoin join, JoinInputNode.InputType inputTypeOnLeft, JoinInputNode.InputType inputTypeOnRight) {
JoinRelType joinRelType = join.getJoinType();
if (joinRelType.compareTo(JoinRelType.INNER) != 0 && joinRelType.compareTo(JoinRelType.LEFT) != 0 && joinRelType.compareTo(JoinRelType.RIGHT) != 0) {
throw new SamzaException("Query with only INNER and LEFT/RIGHT OUTER join are supported.");
}
boolean isTablePosOnLeft = inputTypeOnLeft != JoinInputNode.InputType.STREAM;
boolean isTablePosOnRight = inputTypeOnRight != JoinInputNode.InputType.STREAM;
if (!isTablePosOnLeft && !isTablePosOnRight) {
throw new SamzaException("Invalid query with both sides of join being denoted as 'stream'. " + "Stream-stream join is not yet supported. " + dumpRelPlanForNode(join));
}
if (isTablePosOnLeft && isTablePosOnRight) {
throw new SamzaException("Invalid query with both sides of join being denoted as 'table'. " + dumpRelPlanForNode(join));
}
if (joinRelType.compareTo(JoinRelType.LEFT) == 0 && isTablePosOnLeft) {
throw new SamzaException("Invalid query for outer left join. Left side of the join should be a 'stream' and " + "right side of join should be a 'table'. " + dumpRelPlanForNode(join));
}
if (joinRelType.compareTo(JoinRelType.RIGHT) == 0 && isTablePosOnRight) {
throw new SamzaException("Invalid query for outer right join. Left side of the join should be a 'table' and " + "right side of join should be a 'stream'. " + dumpRelPlanForNode(join));
}
final List<RexNode> conjunctionList = new ArrayList<>();
decomposeAndValidateConjunction(join.getCondition(), conjunctionList);
if (conjunctionList.isEmpty()) {
throw new SamzaException("Query results in a cross join, which is not supported. Please optimize the query." + " It is expected that the joins should include JOIN ON operator in the sql query.");
}
// TODO Not sure why we can not allow literal as part of the join condition will revisit this in another scope
conjunctionList.forEach(rexNode -> rexNode.accept(new RexShuttle() {
@Override
public RexNode visitLiteral(RexLiteral literal) {
throw new SamzaException("Join Condition can not allow literal " + literal.toString() + " join node" + join.getDigest());
}
}));
final JoinInputNode.InputType rootTableInput = isTablePosOnRight ? inputTypeOnRight : inputTypeOnLeft;
if (rootTableInput.compareTo(JoinInputNode.InputType.REMOTE_TABLE) != 0) {
// it is not a remote table all is good we do not have to validate the project on key Column
return;
}
/*
For remote Table we need to validate The join Condition and The project that is above remote table scan.
- As of today Filter need to be exactly one equi-join using the __key__ column (see SAMZA-2554)
- The Project on the top of the remote table has to contain only simple input references to any of the column used in the join.
*/
// First let's collect the ref of columns used by the join condition.
List<RexInputRef> refCollector = new ArrayList<>();
join.getCondition().accept(new RexShuttle() {
@Override
public RexNode visitInputRef(RexInputRef inputRef) {
refCollector.add(inputRef);
return inputRef;
}
});
// start index of the Remote table within the Join Row
final int tableStartIndex = isTablePosOnRight ? join.getLeft().getRowType().getFieldCount() : 0;
// end index of the Remote table withing the Join Row
final int tableEndIndex = isTablePosOnRight ? join.getRowType().getFieldCount() : join.getLeft().getRowType().getFieldCount();
List<Integer> tableRefsIdx = refCollector.stream().map(x -> x.getIndex()).filter(// collect all the refs form table side
x -> tableStartIndex <= x && x < tableEndIndex).map(// re-adjust the offset
x -> x - tableStartIndex).sorted().collect(// we have a list with all the input from table side with 0 based index.
Collectors.toList());
if (conjunctionList.size() != 1 || tableRefsIdx.size() != 1) {
// TODO We can relax this by allowing another filter to be evaluated post lookup see SAMZA-2554
throw new SamzaException("Invalid query for join condition must contain exactly one predicate for remote table on __key__ column " + dumpRelPlanForNode(join));
}
// Validate the Project, follow each input and ensure that it is a simple ref with no rexCall in the way.
if (!isValidRemoteJoinRef(tableRefsIdx.get(0), isTablePosOnRight ? join.getRight() : join.getLeft())) {
throw new SamzaException("Invalid query for join condition can not have an expression and must be reference " + SamzaSqlRelMessage.KEY_NAME + " column " + dumpRelPlanForNode(join));
}
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral in project flink by apache.
the class SqlFunctionConverter method visitOver.
@Override
public RexNode visitOver(RexOver over) {
SqlOperator operator = convertOperator(over.getAggOperator());
Preconditions.checkArgument(operator instanceof SqlAggFunction, "Expect converted operator to be an agg function, but got " + operator.toString());
SqlAggFunction convertedAgg = (SqlAggFunction) operator;
RexWindow window = over.getWindow();
// let's not rely on the type of the RexOver created by Hive since it can be different from
// what Flink expects
RelDataType inferredType = builder.makeCall(convertedAgg, over.getOperands()).getType();
// Hive may add literals to partition keys, remove them
List<RexNode> partitionKeys = new ArrayList<>();
for (RexNode hivePartitionKey : getPartKeys(window)) {
if (!(hivePartitionKey instanceof RexLiteral)) {
partitionKeys.add(hivePartitionKey);
}
}
List<RexFieldCollation> convertedOrderKeys = new ArrayList<>(getOrderKeys(window).size());
for (RexFieldCollation orderKey : getOrderKeys(window)) {
convertedOrderKeys.add(new RexFieldCollation(orderKey.getKey().accept(this), orderKey.getValue()));
}
final boolean[] update = null;
return HiveParserUtils.makeOver(builder, inferredType, convertedAgg, visitList(over.getOperands(), update), visitList(partitionKeys, update), convertedOrderKeys, window.getLowerBound(), window.getUpperBound(), window.isRows(), true, false, false, false);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral in project calcite by apache.
the class EnumerableValues method implement.
public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
/*
return Linq4j.asEnumerable(
new Object[][] {
new Object[] {1, 2},
new Object[] {3, 4}
});
*/
final JavaTypeFactory typeFactory = (JavaTypeFactory) getCluster().getTypeFactory();
final BlockBuilder builder = new BlockBuilder();
final PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferCustom());
final Type rowClass = physType.getJavaRowType();
final List<Expression> expressions = new ArrayList<Expression>();
final List<RelDataTypeField> fields = rowType.getFieldList();
for (List<RexLiteral> tuple : tuples) {
final List<Expression> literals = new ArrayList<Expression>();
for (Pair<RelDataTypeField, RexLiteral> pair : Pair.zip(fields, tuple)) {
literals.add(RexToLixTranslator.translateLiteral(pair.right, pair.left.getType(), typeFactory, RexImpTable.NullAs.NULL));
}
expressions.add(physType.record(literals));
}
builder.add(Expressions.return_(null, Expressions.call(BuiltInMethod.AS_ENUMERABLE.method, Expressions.newArrayInit(Primitive.box(rowClass), expressions))));
return implementor.result(physType, builder.toBlock());
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral in project calcite by apache.
the class ElasticsearchRules method isItem.
/**
* Returns 'string' if it is a call to item['string'], null otherwise.
*/
static String isItem(RexCall call) {
if (call.getOperator() != SqlStdOperatorTable.ITEM) {
return null;
}
final RexNode op0 = call.getOperands().get(0);
final RexNode op1 = call.getOperands().get(1);
if (op0 instanceof RexInputRef && ((RexInputRef) op0).getIndex() == 0 && op1 instanceof RexLiteral && ((RexLiteral) op1).getValue2() instanceof String) {
return (String) ((RexLiteral) op1).getValue2();
}
return null;
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral in project calcite by apache.
the class EnumerableWindow method implement.
public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
final JavaTypeFactory typeFactory = implementor.getTypeFactory();
final EnumerableRel child = (EnumerableRel) getInput();
final BlockBuilder builder = new BlockBuilder();
final Result result = implementor.visitChild(this, 0, child, pref);
Expression source_ = builder.append("source", result.block);
final List<Expression> translatedConstants = new ArrayList<Expression>(constants.size());
for (RexLiteral constant : constants) {
translatedConstants.add(RexToLixTranslator.translateLiteral(constant, constant.getType(), typeFactory, RexImpTable.NullAs.NULL));
}
PhysType inputPhysType = result.physType;
final int w = implementor.windowCount++;
ParameterExpression prevStart = Expressions.parameter(int.class, builder.newName("prevStart" + w));
ParameterExpression prevEnd = Expressions.parameter(int.class, builder.newName("prevEnd" + w));
builder.add(Expressions.declare(0, prevStart, null));
builder.add(Expressions.declare(0, prevEnd, null));
for (int windowIdx = 0; windowIdx < groups.size(); windowIdx++) {
Group group = groups.get(windowIdx);
// Comparator:
// final Comparator<JdbcTest.Employee> comparator =
// new Comparator<JdbcTest.Employee>() {
// public int compare(JdbcTest.Employee o1,
// JdbcTest.Employee o2) {
// return Integer.compare(o1.empid, o2.empid);
// }
// };
final Expression comparator_ = builder.append("comparator", inputPhysType.generateComparator(group.collation()));
Pair<Expression, Expression> partitionIterator = getPartitionIterator(builder, source_, inputPhysType, group, comparator_);
final Expression collectionExpr = partitionIterator.left;
final Expression iterator_ = partitionIterator.right;
List<AggImpState> aggs = new ArrayList<AggImpState>();
List<AggregateCall> aggregateCalls = group.getAggregateCalls(this);
for (int aggIdx = 0; aggIdx < aggregateCalls.size(); aggIdx++) {
AggregateCall call = aggregateCalls.get(aggIdx);
aggs.add(new AggImpState(aggIdx, call, true));
}
// The output from this stage is the input plus the aggregate functions.
final RelDataTypeFactory.Builder typeBuilder = typeFactory.builder();
typeBuilder.addAll(inputPhysType.getRowType().getFieldList());
for (AggImpState agg : aggs) {
typeBuilder.add(agg.call.name, agg.call.type);
}
RelDataType outputRowType = typeBuilder.build();
final PhysType outputPhysType = PhysTypeImpl.of(typeFactory, outputRowType, pref.prefer(result.format));
final Expression list_ = builder.append("list", Expressions.new_(ArrayList.class, Expressions.call(collectionExpr, BuiltInMethod.COLLECTION_SIZE.method)), false);
Pair<Expression, Expression> collationKey = getRowCollationKey(builder, inputPhysType, group, windowIdx);
Expression keySelector = collationKey.left;
Expression keyComparator = collationKey.right;
final BlockBuilder builder3 = new BlockBuilder();
final Expression rows_ = builder3.append("rows", Expressions.convert_(Expressions.call(iterator_, BuiltInMethod.ITERATOR_NEXT.method), Object[].class), false);
builder3.add(Expressions.statement(Expressions.assign(prevStart, Expressions.constant(-1))));
builder3.add(Expressions.statement(Expressions.assign(prevEnd, Expressions.constant(Integer.MAX_VALUE))));
final BlockBuilder builder4 = new BlockBuilder();
final ParameterExpression i_ = Expressions.parameter(int.class, builder4.newName("i"));
final Expression row_ = builder4.append("row", RexToLixTranslator.convert(Expressions.arrayIndex(rows_, i_), inputPhysType.getJavaRowType()));
final RexToLixTranslator.InputGetter inputGetter = new WindowRelInputGetter(row_, inputPhysType, result.physType.getRowType().getFieldCount(), translatedConstants);
final RexToLixTranslator translator = RexToLixTranslator.forAggregation(typeFactory, builder4, inputGetter);
final List<Expression> outputRow = new ArrayList<Expression>();
int fieldCountWithAggResults = inputPhysType.getRowType().getFieldCount();
for (int i = 0; i < fieldCountWithAggResults; i++) {
outputRow.add(inputPhysType.fieldReference(row_, i, outputPhysType.getJavaFieldType(i)));
}
declareAndResetState(typeFactory, builder, result, windowIdx, aggs, outputPhysType, outputRow);
// There are assumptions that minX==0. If ever change this, look for
// frameRowCount, bounds checking, etc
final Expression minX = Expressions.constant(0);
final Expression partitionRowCount = builder3.append("partRows", Expressions.field(rows_, "length"));
final Expression maxX = builder3.append("maxX", Expressions.subtract(partitionRowCount, Expressions.constant(1)));
final Expression startUnchecked = builder4.append("start", translateBound(translator, i_, row_, minX, maxX, rows_, group, true, inputPhysType, comparator_, keySelector, keyComparator));
final Expression endUnchecked = builder4.append("end", translateBound(translator, i_, row_, minX, maxX, rows_, group, false, inputPhysType, comparator_, keySelector, keyComparator));
final Expression startX;
final Expression endX;
final Expression hasRows;
if (group.isAlwaysNonEmpty()) {
startX = startUnchecked;
endX = endUnchecked;
hasRows = Expressions.constant(true);
} else {
Expression startTmp = group.lowerBound.isUnbounded() || startUnchecked == i_ ? startUnchecked : builder4.append("startTmp", Expressions.call(null, BuiltInMethod.MATH_MAX.method, startUnchecked, minX));
Expression endTmp = group.upperBound.isUnbounded() || endUnchecked == i_ ? endUnchecked : builder4.append("endTmp", Expressions.call(null, BuiltInMethod.MATH_MIN.method, endUnchecked, maxX));
ParameterExpression startPe = Expressions.parameter(0, int.class, builder4.newName("startChecked"));
ParameterExpression endPe = Expressions.parameter(0, int.class, builder4.newName("endChecked"));
builder4.add(Expressions.declare(Modifier.FINAL, startPe, null));
builder4.add(Expressions.declare(Modifier.FINAL, endPe, null));
hasRows = builder4.append("hasRows", Expressions.lessThanOrEqual(startTmp, endTmp));
builder4.add(Expressions.ifThenElse(hasRows, Expressions.block(Expressions.statement(Expressions.assign(startPe, startTmp)), Expressions.statement(Expressions.assign(endPe, endTmp))), Expressions.block(Expressions.statement(Expressions.assign(startPe, Expressions.constant(-1))), Expressions.statement(Expressions.assign(endPe, Expressions.constant(-1))))));
startX = startPe;
endX = endPe;
}
final BlockBuilder builder5 = new BlockBuilder(true, builder4);
BinaryExpression rowCountWhenNonEmpty = Expressions.add(startX == minX ? endX : Expressions.subtract(endX, startX), Expressions.constant(1));
final Expression frameRowCount;
if (hasRows.equals(Expressions.constant(true))) {
frameRowCount = builder4.append("totalRows", rowCountWhenNonEmpty);
} else {
frameRowCount = builder4.append("totalRows", Expressions.condition(hasRows, rowCountWhenNonEmpty, Expressions.constant(0)));
}
ParameterExpression actualStart = Expressions.parameter(0, int.class, builder5.newName("actualStart"));
final BlockBuilder builder6 = new BlockBuilder(true, builder5);
builder6.add(Expressions.statement(Expressions.assign(actualStart, startX)));
for (final AggImpState agg : aggs) {
agg.implementor.implementReset(agg.context, new WinAggResetContextImpl(builder6, agg.state, i_, startX, endX, hasRows, partitionRowCount, frameRowCount));
}
Expression lowerBoundCanChange = group.lowerBound.isUnbounded() && group.lowerBound.isPreceding() ? Expressions.constant(false) : Expressions.notEqual(startX, prevStart);
Expression needRecomputeWindow = Expressions.orElse(lowerBoundCanChange, Expressions.lessThan(endX, prevEnd));
BlockStatement resetWindowState = builder6.toBlock();
if (resetWindowState.statements.size() == 1) {
builder5.add(Expressions.declare(0, actualStart, Expressions.condition(needRecomputeWindow, startX, Expressions.add(prevEnd, Expressions.constant(1)))));
} else {
builder5.add(Expressions.declare(0, actualStart, null));
builder5.add(Expressions.ifThenElse(needRecomputeWindow, resetWindowState, Expressions.statement(Expressions.assign(actualStart, Expressions.add(prevEnd, Expressions.constant(1))))));
}
if (lowerBoundCanChange instanceof BinaryExpression) {
builder5.add(Expressions.statement(Expressions.assign(prevStart, startX)));
}
builder5.add(Expressions.statement(Expressions.assign(prevEnd, endX)));
final BlockBuilder builder7 = new BlockBuilder(true, builder5);
final DeclarationStatement jDecl = Expressions.declare(0, "j", actualStart);
final PhysType inputPhysTypeFinal = inputPhysType;
final Function<BlockBuilder, WinAggFrameResultContext> resultContextBuilder = getBlockBuilderWinAggFrameResultContextFunction(typeFactory, result, translatedConstants, comparator_, rows_, i_, startX, endX, minX, maxX, hasRows, frameRowCount, partitionRowCount, jDecl, inputPhysTypeFinal);
final Function<AggImpState, List<RexNode>> rexArguments = new Function<AggImpState, List<RexNode>>() {
public List<RexNode> apply(AggImpState agg) {
List<Integer> argList = agg.call.getArgList();
List<RelDataType> inputTypes = EnumUtils.fieldRowTypes(result.physType.getRowType(), constants, argList);
List<RexNode> args = new ArrayList<RexNode>(inputTypes.size());
for (int i = 0; i < argList.size(); i++) {
Integer idx = argList.get(i);
args.add(new RexInputRef(idx, inputTypes.get(i)));
}
return args;
}
};
implementAdd(aggs, builder7, resultContextBuilder, rexArguments, jDecl);
BlockStatement forBlock = builder7.toBlock();
if (!forBlock.statements.isEmpty()) {
// For instance, row_number does not use for loop to compute the value
Statement forAggLoop = Expressions.for_(Arrays.asList(jDecl), Expressions.lessThanOrEqual(jDecl.parameter, endX), Expressions.preIncrementAssign(jDecl.parameter), forBlock);
if (!hasRows.equals(Expressions.constant(true))) {
forAggLoop = Expressions.ifThen(hasRows, forAggLoop);
}
builder5.add(forAggLoop);
}
if (implementResult(aggs, builder5, resultContextBuilder, rexArguments, true)) {
builder4.add(Expressions.ifThen(Expressions.orElse(lowerBoundCanChange, Expressions.notEqual(endX, prevEnd)), builder5.toBlock()));
}
implementResult(aggs, builder4, resultContextBuilder, rexArguments, false);
builder4.add(Expressions.statement(Expressions.call(list_, BuiltInMethod.COLLECTION_ADD.method, outputPhysType.record(outputRow))));
builder3.add(Expressions.for_(Expressions.declare(0, i_, Expressions.constant(0)), Expressions.lessThan(i_, Expressions.field(rows_, "length")), Expressions.preIncrementAssign(i_), builder4.toBlock()));
builder.add(Expressions.while_(Expressions.call(iterator_, BuiltInMethod.ITERATOR_HAS_NEXT.method), builder3.toBlock()));
builder.add(Expressions.statement(Expressions.call(collectionExpr, BuiltInMethod.MAP_CLEAR.method)));
// We're not assigning to "source". For each group, create a new
// final variable called "source" or "sourceN".
source_ = builder.append("source", Expressions.call(BuiltInMethod.AS_ENUMERABLE.method, list_));
inputPhysType = outputPhysType;
}
// return Linq4j.asEnumerable(list);
builder.add(Expressions.return_(null, source_));
return implementor.result(inputPhysType, builder.toBlock());
}
Aggregations