Examples with TableSpec org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec used on opensource projects

Example 1 with TableSpec

use of org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec in project hive by apache.

the class StatsTask method getPartitionsList.

/**
   * Get the list of partitions that need to update statistics.
   * TODO: we should reuse the Partitions generated at compile time
   * since getting the list of partitions is quite expensive.
   *
   * @return a list of partitions that need to update statistics.
   * @throws HiveException
   */
private List<Partition> getPartitionsList(Hive db) throws HiveException {
    if (work.getLoadFileDesc() != null) {
        //we are in CTAS, so we know there are no partitions
        return null;
    }
    List<Partition> list = new ArrayList<Partition>();
    if (work.getTableSpecs() != null) {
        // ANALYZE command
        TableSpec tblSpec = work.getTableSpecs();
        table = tblSpec.tableHandle;
        if (!table.isPartitioned()) {
            return null;
        }
        // get all partitions that matches with the partition spec
        List<Partition> partitions = tblSpec.partitions;
        if (partitions != null) {
            for (Partition partn : partitions) {
                list.add(partn);
            }
        }
    } else if (work.getLoadTableDesc() != null) {
        // INSERT OVERWRITE command
        LoadTableDesc tbd = work.getLoadTableDesc();
        table = db.getTable(tbd.getTable().getTableName());
        if (!table.isPartitioned()) {
            return null;
        }
        DynamicPartitionCtx dpCtx = tbd.getDPCtx();
        if (dpCtx != null && dpCtx.getNumDPCols() > 0) {
            // If no dynamic partitions are generated, dpPartSpecs may not be initialized
            if (dpPartSpecs != null) {
                // load the list of DP partitions and return the list of partition specs
                list.addAll(dpPartSpecs);
            }
        } else {
            // static partition
            Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
            list.add(partn);
        }
    }
    return list;
}

Example 2 with TableSpec

use of org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec in project hive by apache.

the class GenMapRedUtils method getConfirmedPartitionsForScan.

public static Set<Partition> getConfirmedPartitionsForScan(TableScanOperator tableScanOp) {
    Set<Partition> confirmedPartns = new HashSet<Partition>();
    TableSpec tblSpec = tableScanOp.getConf().getTableMetadata().getTableSpec();
    if (tblSpec.specType == TableSpec.SpecType.STATIC_PARTITION) {
        // static partition
        if (tblSpec.partHandle != null) {
            confirmedPartns.add(tblSpec.partHandle);
        } else {
            // partial partition spec has null partHandle
            confirmedPartns.addAll(tblSpec.partitions);
        }
    } else if (tblSpec.specType == TableSpec.SpecType.DYNAMIC_PARTITION) {
        // dynamic partition
        confirmedPartns.addAll(tblSpec.partitions);
    }
    return confirmedPartns;
}

Example 3 with TableSpec

use of org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec in project hive by apache.

the class StatsNoJobTask method getPartitionsList.

private List<Partition> getPartitionsList() throws HiveException {
    if (work.getTableSpecs() != null) {
        TableSpec tblSpec = work.getTableSpecs();
        table = tblSpec.tableHandle;
        if (!table.isPartitioned()) {
            return null;
        } else {
            return tblSpec.partitions;
        }
    }
    return null;
}

Example 4 with TableSpec

use of org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec in project hive by apache.

the class ProcessAnalyzeTable method genTableStats.

private Task<?> genTableStats(GenTezProcContext context, TableScanOperator tableScan) throws HiveException {
    Class<? extends InputFormat> inputFormat = tableScan.getConf().getTableMetadata().getInputFormatClass();
    ParseContext parseContext = context.parseContext;
    Table table = tableScan.getConf().getTableMetadata();
    List<Partition> partitions = new ArrayList<>();
    if (table.isPartitioned()) {
        partitions.addAll(parseContext.getPrunedPartitions(tableScan).getPartitions());
        for (Partition partn : partitions) {
            LOG.debug("XXX: adding part: " + partn);
            context.outputs.add(new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK));
        }
    }
    TableSpec tableSpec = new TableSpec(table, partitions);
    tableScan.getConf().getTableMetadata().setTableSpec(tableSpec);
    if (inputFormat.equals(OrcInputFormat.class)) {
        // For ORC, there is no Tez Job for table stats.
        StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec());
        snjWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
        // If partition is specified, get pruned partition list
        if (partitions.size() > 0) {
            snjWork.setPrunedPartitionList(parseContext.getPrunedPartitions(tableScan));
        }
        return TaskFactory.get(snjWork, parseContext.getConf());
    } else {
        StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec());
        statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix());
        statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir());
        statsWork.setSourceTask(context.currentTask);
        statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
        return TaskFactory.get(statsWork, parseContext.getConf());
    }
}