use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project drill by apache.
the class InfoSchemaRecordGenerator method visitTables.
/**
* Visit the tables in the given schema. The
* @param schemaPath the path to the given schema
* @param schema the given schema
*/
public void visitTables(String schemaPath, SchemaPlus schema) {
final AbstractSchema drillSchema = schema.unwrap(AbstractSchema.class);
final List<String> tableNames = Lists.newArrayList(schema.getTableNames());
for (Pair<String, ? extends Table> tableNameToTable : drillSchema.getTablesByNames(tableNames)) {
final String tableName = tableNameToTable.getKey();
final Table table = tableNameToTable.getValue();
final TableType tableType = table.getJdbcTableType();
// Visit the table, and if requested ...
if (shouldVisitTable(schemaPath, tableName, tableType) && visitTable(schemaPath, tableName, table)) {
// ... do for each of the table's fields.
final RelDataType tableRow = table.getRowType(new JavaTypeFactoryImpl());
for (RelDataTypeField field : tableRow.getFieldList()) {
if (shouldVisitColumn(schemaPath, tableName, field.getName())) {
visitField(schemaPath, tableName, field);
}
}
}
}
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project drill by apache.
the class InsertLocalExchangeVisitor method visitExchange.
@Override
public Prel visitExchange(ExchangePrel prel, Void value) throws RuntimeException {
Prel child = ((Prel) prel.getInput()).accept(this, null);
// If DeMuxExchange is enabled, insert a UnorderedDeMuxExchangePrel after HashToRandomExchangePrel.
if (!(prel instanceof HashToRandomExchangePrel)) {
return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList(((RelNode) child)));
}
Prel newPrel = child;
final HashToRandomExchangePrel hashPrel = (HashToRandomExchangePrel) prel;
final List<String> childFields = child.getRowType().getFieldNames();
List<RexNode> removeUpdatedExpr = null;
if (isMuxEnabled) {
// Insert Project Operator with new column that will be a hash for HashToRandomExchange fields
final List<DistributionField> distFields = hashPrel.getFields();
final List<String> outputFieldNames = Lists.newArrayList(childFields);
final RexBuilder rexBuilder = prel.getCluster().getRexBuilder();
final List<RelDataTypeField> childRowTypeFields = child.getRowType().getFieldList();
final HashExpressionCreatorHelper<RexNode> hashHelper = new RexNodeBasedHashExpressionCreatorHelper(rexBuilder);
final List<RexNode> distFieldRefs = Lists.newArrayListWithExpectedSize(distFields.size());
for (int i = 0; i < distFields.size(); i++) {
final int fieldId = distFields.get(i).getFieldId();
distFieldRefs.add(rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId));
}
final List<RexNode> updatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size());
removeUpdatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size());
for (RelDataTypeField field : childRowTypeFields) {
RexNode rex = rexBuilder.makeInputRef(field.getType(), field.getIndex());
updatedExpr.add(rex);
removeUpdatedExpr.add(rex);
}
outputFieldNames.add(HashPrelUtil.HASH_EXPR_NAME);
// distribution seed
final RexNode distSeed = rexBuilder.makeBigintLiteral(BigDecimal.valueOf(HashPrelUtil.DIST_SEED));
updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, distSeed, hashHelper));
RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), updatedExpr, outputFieldNames);
ProjectPrel addColumnprojectPrel = new ProjectPrel(child.getCluster(), child.getTraitSet(), child, updatedExpr, rowType);
newPrel = new UnorderedMuxExchangePrel(addColumnprojectPrel.getCluster(), addColumnprojectPrel.getTraitSet(), addColumnprojectPrel);
}
newPrel = new HashToRandomExchangePrel(prel.getCluster(), prel.getTraitSet(), newPrel, ((HashToRandomExchangePrel) prel).getFields());
if (isDeMuxEnabled) {
HashToRandomExchangePrel hashExchangePrel = (HashToRandomExchangePrel) newPrel;
// Insert a DeMuxExchange to narrow down the number of receivers
newPrel = new UnorderedDeMuxExchangePrel(prel.getCluster(), prel.getTraitSet(), hashExchangePrel, hashExchangePrel.getFields());
}
if (isMuxEnabled) {
// remove earlier inserted Project Operator - since it creates issues down the road in HashJoin
RelDataType removeRowType = RexUtil.createStructType(newPrel.getCluster().getTypeFactory(), removeUpdatedExpr, childFields);
ProjectPrel removeColumnProjectPrel = new ProjectPrel(newPrel.getCluster(), newPrel.getTraitSet(), newPrel, removeUpdatedExpr, removeRowType);
return removeColumnProjectPrel;
}
return newPrel;
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project hive by apache.
the class HiveRelDecorrelator method decorrelateRel.
/**
* Rewrite LogicalProject.
*
* @param rel the project rel to rewrite
*/
public Frame decorrelateRel(LogicalProject rel) throws SemanticException {
//
// Rewrite logic:
//
// 1. Pass along any correlated variables coming from the input.
//
final RelNode oldInput = rel.getInput();
Frame frame = getInvoke(oldInput, rel);
if (frame == null) {
// If input has not been rewritten, do not rewrite this rel.
return null;
}
final List<RexNode> oldProjects = rel.getProjects();
final List<RelDataTypeField> relOutput = rel.getRowType().getFieldList();
// LogicalProject projects the original expressions,
// plus any correlated variables the input wants to pass along.
final List<Pair<RexNode, String>> projects = Lists.newArrayList();
// and produce the correlated variables in the new output.
if (cm.mapRefRelToCorRef.containsKey(rel)) {
frame = decorrelateInputWithValueGenerator(rel);
}
// LogicalProject projects the original expressions
final Map<Integer, Integer> mapOldToNewOutputs = new HashMap<>();
int newPos;
for (newPos = 0; newPos < oldProjects.size(); newPos++) {
projects.add(newPos, Pair.of(decorrelateExpr(oldProjects.get(newPos)), relOutput.get(newPos).getName()));
mapOldToNewOutputs.put(newPos, newPos);
}
// Project any correlated variables the input wants to pass along.
final SortedMap<CorDef, Integer> corDefOutputs = new TreeMap<>();
for (Map.Entry<CorDef, Integer> entry : frame.corDefOutputs.entrySet()) {
projects.add(RexInputRef.of2(entry.getValue(), frame.r.getRowType().getFieldList()));
corDefOutputs.put(entry.getKey(), newPos);
newPos++;
}
RelNode newProject = HiveProject.create(frame.r, Pair.left(projects), Pair.right(projects));
return register(rel, newProject, mapOldToNewOutputs, corDefOutputs);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project hive by apache.
the class HiveRelDecorrelator method projectJoinOutputWithNullability.
/**
* Pulls project above the join from its RHS input. Enforces nullability
* for join output.
*
* @param join Join
* @param project Original project as the right-hand input of the join
* @param nullIndicatorPos Position of null indicator
* @return the subtree with the new LogicalProject at the root
*/
private RelNode projectJoinOutputWithNullability(LogicalJoin join, LogicalProject project, int nullIndicatorPos) {
final RelDataTypeFactory typeFactory = join.getCluster().getTypeFactory();
final RelNode left = join.getLeft();
final JoinRelType joinType = join.getJoinType();
RexInputRef nullIndicator = new RexInputRef(nullIndicatorPos, typeFactory.createTypeWithNullability(join.getRowType().getFieldList().get(nullIndicatorPos).getType(), true));
// now create the new project
List<Pair<RexNode, String>> newProjExprs = Lists.newArrayList();
// project everything from the LHS and then those from the original
// projRel
List<RelDataTypeField> leftInputFields = left.getRowType().getFieldList();
for (int i = 0; i < leftInputFields.size(); i++) {
newProjExprs.add(RexInputRef.of2(i, leftInputFields));
}
// Marked where the projected expr is coming from so that the types will
// become nullable for the original projections which are now coming out
// of the nullable side of the OJ.
boolean projectPulledAboveLeftCorrelator = joinType.generatesNullsOnRight();
for (Pair<RexNode, String> pair : project.getNamedProjects()) {
RexNode newProjExpr = removeCorrelationExpr(pair.left, projectPulledAboveLeftCorrelator, nullIndicator);
newProjExprs.add(Pair.of(newProjExpr, pair.right));
}
return RelOptUtil.createProject(join, newProjExprs, false);
}
use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeField in project hive by apache.
the class HiveSortLimitPullUpConstantsRule method onMatch.
@Override
public void onMatch(RelOptRuleCall call) {
final RelNode parent = call.rel(0);
final Sort sort = call.rel(1);
final int count = sort.getInput().getRowType().getFieldCount();
if (count == 1) {
// Project operator.
return;
}
final RexBuilder rexBuilder = sort.getCluster().getRexBuilder();
final RelMetadataQuery mq = call.getMetadataQuery();
final RelOptPredicateList predicates = mq.getPulledUpPredicates(sort.getInput());
if (predicates == null) {
return;
}
Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
Map<RexNode, RexNode> constants = new HashMap<>();
for (int i = 0; i < count; i++) {
RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i);
if (conditionsExtracted.containsKey(expr)) {
constants.put(expr, conditionsExtracted.get(expr));
}
}
// None of the expressions are constant. Nothing to do.
if (constants.isEmpty()) {
return;
}
if (count == constants.size()) {
// At least a single item in project is required.
constants.remove(constants.keySet().iterator().next());
}
// Create expressions for Project operators before and after the Sort
List<RelDataTypeField> fields = sort.getInput().getRowType().getFieldList();
List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
List<RexNode> topChildExprs = new ArrayList<>();
List<String> topChildExprsFields = new ArrayList<>();
for (int i = 0; i < count; i++) {
RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i);
RelDataTypeField field = fields.get(i);
if (constants.containsKey(expr)) {
topChildExprs.add(constants.get(expr));
topChildExprsFields.add(field.getName());
} else {
newChildExprs.add(Pair.<RexNode, String>of(expr, field.getName()));
topChildExprs.add(expr);
topChildExprsFields.add(field.getName());
}
}
// Update field collations
final Mappings.TargetMapping mapping = RelOptUtil.permutation(Pair.left(newChildExprs), sort.getInput().getRowType()).inverse();
List<RelFieldCollation> fieldCollations = new ArrayList<>();
for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
final int target = mapping.getTargetOpt(fc.getFieldIndex());
if (target < 0) {
// It is a constant, we can ignore it
continue;
}
fieldCollations.add(fc.copy(target));
}
// Update top Project positions
topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
// Create new Project-Sort-Project sequence
final RelBuilder relBuilder = call.builder();
relBuilder.push(sort.getInput());
relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
final ImmutableList<RexNode> sortFields = relBuilder.fields(RelCollations.of(fieldCollations));
relBuilder.sortLimit(sort.offset == null ? -1 : RexLiteral.intValue(sort.offset), sort.fetch == null ? -1 : RexLiteral.intValue(sort.fetch), sortFields);
// Create top Project fixing nullability of fields
relBuilder.project(topChildExprs, topChildExprsFields);
relBuilder.convert(sort.getRowType(), false);
List<RelNode> inputs = new ArrayList<>();
for (RelNode child : parent.getInputs()) {
if (!((HepRelVertex) child).getCurrentRel().equals(sort)) {
inputs.add(child);
} else {
inputs.add(relBuilder.build());
}
}
call.transformTo(parent.copy(parent.getTraitSet(), inputs));
}
Aggregations