Examples with HiveException org.apache.hadoop.hive.ql.metadata.HiveException used on opensource projects

Example 31 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class IndexUtils method isIndexPartitionFresh.

/**
   * Check the index partitions on a partitioned table exist and are fresh
   */
private static boolean isIndexPartitionFresh(Hive hive, Index index, Partition part) throws HiveException {
    LOG.info("checking index staleness...");
    try {
        String indexTs = index.getParameters().get(part.getSpec().toString());
        if (indexTs == null) {
            return false;
        }
        FileSystem partFs = part.getDataLocation().getFileSystem(hive.getConf());
        FileStatus[] parts = partFs.listStatus(part.getDataLocation(), FileUtils.HIDDEN_FILES_PATH_FILTER);
        for (FileStatus status : parts) {
            if (status.getModificationTime() > Long.parseLong(indexTs)) {
                LOG.info("Index is stale on partition '" + part.getName() + "'. Modified time (" + status.getModificationTime() + ") for '" + status.getPath() + "' is higher than index creation time (" + indexTs + ").");
                return false;
            }
        }
    } catch (IOException e) {
        throw new HiveException("Failed to grab timestamp information from partition '" + part.getName() + "': " + e.getMessage(), e);
    }
    return true;
}

Example 32 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class IndexUtils method isIndexTableFresh.

/**
   * Check that the indexes on the un-partitioned table exist and are fresh
   */
private static boolean isIndexTableFresh(Hive hive, List<Index> indexes, Table src) throws HiveException {
    //check that they exist
    if (indexes == null || indexes.size() == 0) {
        return false;
    }
    //check that they are not stale
    for (Index index : indexes) {
        LOG.info("checking index staleness...");
        try {
            String indexTs = index.getParameters().get("base_timestamp");
            if (indexTs == null) {
                return false;
            }
            FileSystem srcFs = src.getPath().getFileSystem(hive.getConf());
            FileStatus[] srcs = srcFs.listStatus(src.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
            for (FileStatus status : srcs) {
                if (status.getModificationTime() > Long.parseLong(indexTs)) {
                    LOG.info("Index is stale on table '" + src.getTableName() + "'. Modified time (" + status.getModificationTime() + ") for '" + status.getPath() + "' is higher than index creation time (" + indexTs + ").");
                    return false;
                }
            }
        } catch (IOException e) {
            throw new HiveException("Failed to grab timestamp information from table '" + src.getTableName() + "': " + e.getMessage(), e);
        }
    }
    return true;
}

Example 33 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class RewriteGBUsingIndex method getIndexToKeysMap.

/**
   * This code block iterates over indexes on the table and populates the indexToKeys map
   * for all the indexes that satisfy the rewrite criteria.
   * @param indexTables
   * @return
   * @throws SemanticException
   */
Map<Index, String> getIndexToKeysMap(List<Index> indexTables) throws SemanticException {
    Hive hiveInstance = hiveDb;
    Map<Index, String> indexToKeysMap = new LinkedHashMap<Index, String>();
    for (int idxCtr = 0; idxCtr < indexTables.size(); idxCtr++) {
        Index index = indexTables.get(idxCtr);
        //Getting index key columns
        StorageDescriptor sd = index.getSd();
        List<FieldSchema> idxColList = sd.getCols();
        assert idxColList.size() == 1;
        String indexKeyName = idxColList.get(0).getName();
        // Check that the index schema is as expected. This code block should
        // catch problems of this rewrite breaking when the AggregateIndexHandler
        // index is changed.
        List<String> idxTblColNames = new ArrayList<String>();
        try {
            String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName());
            Table idxTbl = hiveInstance.getTable(qualified[0], qualified[1]);
            for (FieldSchema idxTblCol : idxTbl.getCols()) {
                idxTblColNames.add(idxTblCol.getName());
            }
        } catch (HiveException e) {
            LOG.error("Got exception while locating index table, " + "skipping " + getName() + " optimization");
            LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
            throw new SemanticException(e.getMessage(), e);
        }
        assert (idxTblColNames.contains(IDX_BUCKET_COL));
        assert (idxTblColNames.contains(IDX_OFFSETS_ARRAY_COL));
        // we add all index tables which can be used for rewrite
        // and defer the decision of using a particular index for later
        // this is to allow choosing a index if a better mechanism is
        // designed later to chose a better rewrite
        indexToKeysMap.put(index, indexKeyName);
    }
    return indexToKeysMap;
}

Example 34 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class RewriteGBUsingIndex method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    parseContext = pctx;
    hiveConf = parseContext.getConf();
    try {
        hiveDb = Hive.get(hiveConf);
    } catch (HiveException e) {
        LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
        throw new SemanticException(e.getMessage(), e);
    }
    // Don't try to index optimize the query to build the index
    HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER, false);
    /* Check if the input query passes all the tests to be eligible for a rewrite
     * If yes, rewrite original query; else, return the current parseContext
     */
    if (shouldApplyOptimization()) {
        LOG.info("Rewriting Original Query using " + getName() + " optimization.");
        rewriteOriginalQuery();
    }
    return parseContext;
}

Example 35 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class AnnotateRunTimeStatsOptimizer method setRuntimeStatsDir.

private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException {
    try {
        OperatorDesc conf = op.getConf();
        if (conf != null) {
            LOG.info("setRuntimeStatsDir for " + op.getOperatorId());
            String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString();
            StatsPublisher statsPublisher = new FSStatsPublisher();
            StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf());
            runtimeStatsContext.setStatsTmpDir(path);
            if (!statsPublisher.init(runtimeStatsContext)) {
                LOG.error("StatsPublishing error: StatsPublisher is not initialized.");
                throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg());
            }
            conf.setRuntimeStatsTmpDir(path);
        } else {
            LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null");
        }
    } catch (HiveException e) {
        throw new SemanticException(e);
    }
}