Examples with RelOptHiveTable org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable used on opensource projects

Example 1 with RelOptHiveTable

use of org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable in project hive by apache.

the class HiveOpConverter method visit.

/**
   * TODO: 1. PPD needs to get pushed in to TS
   *
   * @param scanRel
   * @return
   */
OpAttr visit(HiveTableScan scanRel) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + scanRel.getId() + ":" + scanRel.getRelTypeName() + " with row type: [" + scanRel.getRowType() + "]");
    }
    RelOptHiveTable ht = (RelOptHiveTable) scanRel.getTable();
    // 1. Setup TableScan Desc
    // 1.1 Build col details used by scan
    ArrayList<ColumnInfo> colInfos = new ArrayList<ColumnInfo>();
    List<VirtualColumn> virtualCols = new ArrayList<VirtualColumn>();
    List<Integer> neededColumnIDs = new ArrayList<Integer>();
    List<String> neededColumnNames = new ArrayList<String>();
    Set<Integer> vcolsInCalcite = new HashSet<Integer>();
    List<String> partColNames = new ArrayList<String>();
    Map<Integer, VirtualColumn> VColsMap = HiveCalciteUtil.getVColsMap(ht.getVirtualCols(), ht.getNoOfNonVirtualCols());
    Map<Integer, ColumnInfo> posToPartColInfo = ht.getPartColInfoMap();
    Map<Integer, ColumnInfo> posToNonPartColInfo = ht.getNonPartColInfoMap();
    List<Integer> neededColIndxsFrmReloptHT = scanRel.getNeededColIndxsFrmReloptHT();
    List<String> scanColNames = scanRel.getRowType().getFieldNames();
    String tableAlias = scanRel.getConcatQbIDAlias();
    String colName;
    ColumnInfo colInfo;
    VirtualColumn vc;
    for (int index = 0; index < scanRel.getRowType().getFieldList().size(); index++) {
        colName = scanColNames.get(index);
        if (VColsMap.containsKey(index)) {
            vc = VColsMap.get(index);
            virtualCols.add(vc);
            colInfo = new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, true, vc.getIsHidden());
            vcolsInCalcite.add(index);
        } else if (posToPartColInfo.containsKey(index)) {
            partColNames.add(colName);
            colInfo = posToPartColInfo.get(index);
            vcolsInCalcite.add(index);
        } else {
            colInfo = posToNonPartColInfo.get(index);
        }
        colInfos.add(colInfo);
        if (neededColIndxsFrmReloptHT.contains(index)) {
            neededColumnIDs.add(index);
            neededColumnNames.add(colName);
        }
    }
    // 1.2 Create TableScanDesc
    TableScanDesc tsd = new TableScanDesc(tableAlias, virtualCols, ht.getHiveTableMD());
    // 1.3. Set Partition cols in TSDesc
    tsd.setPartColumns(partColNames);
    // 1.4. Set needed cols in TSDesc
    tsd.setNeededColumnIDs(neededColumnIDs);
    tsd.setNeededColumns(neededColumnNames);
    // 2. Setup TableScan
    TableScanOperator ts = (TableScanOperator) OperatorFactory.get(semanticAnalyzer.getOpContext(), tsd, new RowSchema(colInfos));
    // tablescan with same alias.
    if (topOps.get(tableAlias) != null) {
        tableAlias = tableAlias + this.uniqueCounter;
    }
    topOps.put(tableAlias, ts);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated " + ts + " with row schema: [" + ts.getSchema() + "]");
    }
    return new OpAttr(tableAlias, vcolsInCalcite, ts);
}

Example 2 with RelOptHiveTable

use of org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable in project hive by apache.

the class ASTBuilder method table.

public static ASTNode table(RelNode scan) {
    HiveTableScan hts;
    if (scan instanceof DruidQuery) {
        hts = (HiveTableScan) ((DruidQuery) scan).getTableScan();
    } else {
        hts = (HiveTableScan) scan;
    }
    RelOptHiveTable hTbl = (RelOptHiveTable) hts.getTable();
    ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add(ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME").add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName()).add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName()));
    ASTBuilder propList = ASTBuilder.construct(HiveParser.TOK_TABLEPROPLIST, "TOK_TABLEPROPLIST");
    if (scan instanceof DruidQuery) {
        // Pass possible query to Druid
        DruidQuery dq = (DruidQuery) scan;
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_JSON + "\"").add(HiveParser.StringLiteral, "\"" + SemanticAnalyzer.escapeSQLString(dq.getQueryString()) + "\""));
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_TYPE + "\"").add(HiveParser.StringLiteral, "\"" + dq.getQueryType().getQueryName() + "\""));
    }
    if (hts.isInsideView()) {
        // We need to carry the insideView information from calcite into the ast.
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"insideView\"").add(HiveParser.StringLiteral, "\"TRUE\""));
    }
    b.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTIES, "TOK_TABLEPROPERTIES").add(propList));
    // NOTE: Calcite considers tbls to be equal if their names are the same. Hence
    // we need to provide Calcite the fully qualified table name (dbname.tblname)
    // and not the user provided aliases.
    // However in HIVE DB name can not appear in select list; in case of join
    // where table names differ only in DB name, Hive would require user
    // introducing explicit aliases for tbl.
    b.add(HiveParser.Identifier, hts.getTableAlias());
    return b.node();
}

Example 3 with RelOptHiveTable

use of org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable in project hive by apache.

the class FilterSelectivityEstimator method isPartitionPredicate.

private boolean isPartitionPredicate(RexNode expr, RelNode r) {
    if (r instanceof Project) {
        expr = RelOptUtil.pushFilterPastProject(expr, (Project) r);
        return isPartitionPredicate(expr, ((Project) r).getInput());
    } else if (r instanceof Filter) {
        return isPartitionPredicate(expr, ((Filter) r).getInput());
    } else if (r instanceof HiveTableScan) {
        RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScan) r).getTable();
        ImmutableBitSet cols = RelOptUtil.InputFinder.bits(expr);
        return table.containsPartitionColumnsOnly(cols);
    }
    return false;
}

Example 4 with RelOptHiveTable

use of org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable in project hive by apache.

the class HiveRelMdSize method averageColumnSizes.

//~ Methods ----------------------------------------------------------------
public List<Double> averageColumnSizes(HiveTableScan scan, RelMetadataQuery mq) {
    List<Integer> neededcolsLst = scan.getNeededColIndxsFrmReloptHT();
    List<ColStatistics> columnStatistics = ((RelOptHiveTable) scan.getTable()).getColStat(neededcolsLst, true);
    // Obtain list of col stats, or use default if they are not available
    final ImmutableList.Builder<Double> list = ImmutableList.builder();
    int indxRqdCol = 0;
    int nFields = scan.getRowType().getFieldCount();
    for (int i = 0; i < nFields; i++) {
        if (neededcolsLst.contains(i)) {
            ColStatistics columnStatistic = columnStatistics.get(indxRqdCol);
            indxRqdCol++;
            if (columnStatistic == null) {
                RelDataTypeField field = scan.getRowType().getFieldList().get(i);
                list.add(averageTypeValueSize(field.getType()));
            } else {
                list.add(columnStatistic.getAvgColLen());
            }
        } else {
            list.add(new Double(0));
        }
    }
    return list.build();
}

Example 5 with RelOptHiveTable

use of org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable in project hive by apache.

the class HiveRelMdParallelism method splitCount.

public Integer splitCount(HiveTableScan scan, RelMetadataQuery mq) {
    Integer splitCount;
    RelOptHiveTable table = (RelOptHiveTable) scan.getTable();
    List<String> bucketCols = table.getHiveTableMD().getBucketCols();
    if (bucketCols != null && !bucketCols.isEmpty()) {
        splitCount = table.getHiveTableMD().getNumBuckets();
    } else {
        splitCount = splitCountRepartition(scan, mq);
        if (splitCount == null) {
            throw new RuntimeException("Could not get split count for table: " + scan.getTable().getQualifiedName());
        }
    }
    return splitCount;
}