Examples with HiveTableScan org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan used on opensource projects

Example 1 with HiveTableScan

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan in project hive by apache.

the class HiveRelOpMaterializationValidator method visit.

@Override
public RelNode visit(TableScan scan) {
    if (scan instanceof HiveTableScan) {
        HiveTableScan hiveScan = (HiveTableScan) scan;
        RelOptHiveTable relOptHiveTable = (RelOptHiveTable) hiveScan.getTable();
        Table tab = relOptHiveTable.getHiveTableMD();
        if (tab.isTemporary()) {
            fail(tab.getTableName() + " is a temporary table");
        }
        TableType tt = tab.getTableType();
        if (tab.getTableType() == TableType.EXTERNAL_TABLE) {
            fail(tab.getFullyQualifiedName() + " is an external table");
        }
        return scan;
    }
    // TableScan of a non-Hive table - don't support for materializations.
    fail(scan.getTable().getQualifiedName() + " is a table scan of a non-Hive table.");
    return scan;
}

Example 2 with HiveTableScan

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan in project hive by apache.

the class ASTBuilder method table.

public static ASTNode table(final RelNode scan) {
    HiveTableScan hts = null;
    if (scan instanceof HiveJdbcConverter) {
        hts = ((HiveJdbcConverter) scan).getTableScan().getHiveTableScan();
    } else if (scan instanceof DruidQuery) {
        hts = (HiveTableScan) ((DruidQuery) scan).getTableScan();
    } else {
        hts = (HiveTableScan) scan;
    }
    assert hts != null;
    RelOptHiveTable hTbl = (RelOptHiveTable) hts.getTable();
    ASTBuilder tableNameBuilder = ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME").add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName()).add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName());
    if (hTbl.getHiveTableMD().getMetaTable() != null) {
        tableNameBuilder.add(HiveParser.Identifier, hTbl.getHiveTableMD().getMetaTable());
    }
    ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add(tableNameBuilder);
    if (hTbl.getHiveTableMD().getAsOfTimestamp() != null) {
        ASTBuilder asOfBuilder = ASTBuilder.construct(HiveParser.TOK_AS_OF_TIME, "TOK_AS_OF_TIME").add(HiveParser.StringLiteral, hTbl.getHiveTableMD().getAsOfTimestamp());
        b.add(asOfBuilder);
    }
    if (hTbl.getHiveTableMD().getAsOfVersion() != null) {
        ASTBuilder asOfBuilder = ASTBuilder.construct(HiveParser.TOK_AS_OF_VERSION, "TOK_AS_OF_VERSION").add(HiveParser.Number, hTbl.getHiveTableMD().getAsOfVersion());
        b.add(asOfBuilder);
    }
    ASTBuilder propList = ASTBuilder.construct(HiveParser.TOK_TABLEPROPLIST, "TOK_TABLEPROPLIST");
    if (scan instanceof DruidQuery) {
        // Passing query spec, column names and column types to be used as part of Hive Physical execution
        DruidQuery dq = (DruidQuery) scan;
        // Adding Query specs to be used by org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_JSON + "\"").add(HiveParser.StringLiteral, "\"" + SemanticAnalyzer.escapeSQLString(dq.getQueryString()) + "\""));
        // Adding column names used later by org.apache.hadoop.hive.druid.serde.DruidSerDe
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_FIELD_NAMES + "\"").add(HiveParser.StringLiteral, "\"" + dq.getRowType().getFieldNames().stream().map(Object::toString).collect(Collectors.joining(",")) + "\""));
        // Adding column types used later by org.apache.hadoop.hive.druid.serde.DruidSerDe
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_FIELD_TYPES + "\"").add(HiveParser.StringLiteral, "\"" + dq.getRowType().getFieldList().stream().map(e -> TypeConverter.convert(e.getType()).getTypeName()).collect(Collectors.joining(",")) + "\""));
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.DRUID_QUERY_TYPE + "\"").add(HiveParser.StringLiteral, "\"" + dq.getQueryType().getQueryName() + "\""));
    } else if (scan instanceof HiveJdbcConverter) {
        HiveJdbcConverter jdbcConverter = (HiveJdbcConverter) scan;
        final String query = jdbcConverter.generateSql();
        LOGGER.debug("Generated SQL query: " + System.lineSeparator() + query);
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.JDBC_QUERY + "\"").add(HiveParser.StringLiteral, "\"" + SemanticAnalyzer.escapeSQLString(query) + "\""));
        // Whether we can split the query
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.JDBC_SPLIT_QUERY + "\"").add(HiveParser.StringLiteral, "\"" + jdbcConverter.splittingAllowed() + "\""));
        // Adding column names used later by org.apache.hadoop.hive.druid.serde.DruidSerDe
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.JDBC_QUERY_FIELD_NAMES + "\"").add(HiveParser.StringLiteral, "\"" + scan.getRowType().getFieldNames().stream().map(Object::toString).collect(Collectors.joining(",")) + "\""));
        // Adding column types used later by org.apache.hadoop.hive.druid.serde.DruidSerDe
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"" + Constants.JDBC_QUERY_FIELD_TYPES + "\"").add(HiveParser.StringLiteral, "\"" + scan.getRowType().getFieldList().stream().map(e -> TypeConverter.convert(e.getType()).getTypeName()).collect(Collectors.joining(",")) + "\""));
    }
    if (hts.isInsideView()) {
        // We need to carry the insideView information from calcite into the ast.
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, "\"insideView\"").add(HiveParser.StringLiteral, "\"TRUE\""));
    }
    if (hts.getTableScanTrait() != null) {
        // We need to carry the fetchDeletedRows information from calcite into the ast.
        propList.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTY, "TOK_TABLEPROPERTY").add(HiveParser.StringLiteral, String.format("\"%s\"", hts.getTableScanTrait().getPropertyKey())).add(HiveParser.StringLiteral, "\"TRUE\""));
    }
    b.add(ASTBuilder.construct(HiveParser.TOK_TABLEPROPERTIES, "TOK_TABLEPROPERTIES").add(propList));
    // NOTE: Calcite considers tbls to be equal if their names are the same. Hence
    // we need to provide Calcite the fully qualified table name (dbname.tblname)
    // and not the user provided aliases.
    // However in HIVE DB name can not appear in select list; in case of join
    // where table names differ only in DB name, Hive would require user
    // introducing explicit aliases for tbl.
    b.add(HiveParser.Identifier, hts.getTableAlias());
    return b.node();
}

Example 3 with HiveTableScan

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan in project hive by apache.

the class FilterSelectivityEstimator method isPartitionPredicate.

private boolean isPartitionPredicate(RexNode expr, RelNode r) {
    if (r instanceof Project) {
        expr = RelOptUtil.pushFilterPastProject(expr, (Project) r);
        return isPartitionPredicate(expr, ((Project) r).getInput());
    } else if (r instanceof Filter) {
        return isPartitionPredicate(expr, ((Filter) r).getInput());
    } else if (r instanceof HiveTableScan) {
        RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScan) r).getTable();
        ImmutableBitSet cols = RelOptUtil.InputFinder.bits(expr);
        return table.containsPartitionColumnsOnly(cols);
    }
    return false;
}

Example 4 with HiveTableScan

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan in project hive by apache.

the class HiveInsertOnlyScanWriteIdRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    HiveTableScan tableScan = call.rel(0);
    Table tableMD = ((RelOptHiveTable) tableScan.getTable()).getHiveTableMD();
    return !tableMD.isMaterializedView() && AcidUtils.isInsertOnlyTable(tableMD);
}

Example 5 with HiveTableScan

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan in project hive by apache.

the class HiveInsertOnlyScanWriteIdRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    HiveTableScan tableScan = call.rel(0);
    RelNode newTableScan = call.builder().push(tableScan.setTableScanTrait(HiveTableScan.HiveTableScanTrait.FetchInsertOnlyBucketIds)).build();
    call.transformTo(newTableScan);
}