Examples with HiveStorageHandler org.apache.hadoop.hive.ql.metadata.HiveStorageHandler used on opensource projects

Example 16 with HiveStorageHandler

use of org.apache.hadoop.hive.ql.metadata.HiveStorageHandler in project hive by apache.

the class PlanUtils method getTableDesc.

/**
 * Generate a table descriptor from a createTableDesc.
 */
public static TableDesc getTableDesc(CreateTableDesc crtTblDesc, String cols, String colTypes) {
    TableDesc ret;
    // Resolve storage handler (if any)
    try {
        HiveStorageHandler storageHandler = null;
        if (crtTblDesc.getStorageHandler() != null) {
            storageHandler = HiveUtils.getStorageHandler(SessionState.getSessionConf(), crtTblDesc.getStorageHandler());
        }
        Class<? extends Deserializer> serdeClass = getDefaultSerDe();
        String separatorCode = Integer.toString(Utilities.ctrlaCode);
        String columns = cols;
        String columnTypes = colTypes;
        boolean lastColumnTakesRestOfTheLine = false;
        if (storageHandler != null) {
            serdeClass = storageHandler.getSerDeClass();
        } else if (crtTblDesc.getSerName() != null) {
            serdeClass = JavaUtils.loadClass(crtTblDesc.getSerName());
        }
        if (crtTblDesc.getFieldDelim() != null) {
            separatorCode = crtTblDesc.getFieldDelim();
        }
        ret = getTableDesc(serdeClass, separatorCode, columns, columnTypes, lastColumnTakesRestOfTheLine, false);
        // set other table properties
        Properties properties = ret.getProperties();
        if (crtTblDesc.getStorageHandler() != null) {
            properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, crtTblDesc.getStorageHandler());
        }
        if (crtTblDesc.getCollItemDelim() != null) {
            properties.setProperty(serdeConstants.COLLECTION_DELIM, crtTblDesc.getCollItemDelim());
        }
        if (crtTblDesc.getMapKeyDelim() != null) {
            properties.setProperty(serdeConstants.MAPKEY_DELIM, crtTblDesc.getMapKeyDelim());
        }
        if (crtTblDesc.getFieldEscape() != null) {
            properties.setProperty(serdeConstants.ESCAPE_CHAR, crtTblDesc.getFieldEscape());
        }
        if (crtTblDesc.getLineDelim() != null) {
            properties.setProperty(serdeConstants.LINE_DELIM, crtTblDesc.getLineDelim());
        }
        if (crtTblDesc.getNullFormat() != null) {
            properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, crtTblDesc.getNullFormat());
        }
        if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) {
            properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, crtTblDesc.getTableName());
        }
        if (crtTblDesc.getTblProps() != null) {
            properties.putAll(crtTblDesc.getTblProps());
        }
        if (crtTblDesc.getSerdeProps() != null) {
            properties.putAll(crtTblDesc.getSerdeProps());
        }
        // replace the default input & output file format with those found in
        // crtTblDesc
        Class<? extends InputFormat> in_class;
        if (storageHandler != null) {
            in_class = storageHandler.getInputFormatClass();
        } else {
            in_class = JavaUtils.loadClass(crtTblDesc.getInputFormat());
        }
        Class<? extends OutputFormat> out_class;
        if (storageHandler != null) {
            out_class = storageHandler.getOutputFormatClass();
        } else {
            out_class = JavaUtils.loadClass(crtTblDesc.getOutputFormat());
        }
        ret.setInputFileFormatClass(in_class);
        ret.setOutputFileFormatClass(out_class);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException("Unable to find class in getTableDesc: " + e.getMessage(), e);
    } catch (HiveException e) {
        throw new RuntimeException("Error loading storage handler in getTableDesc: " + e.getMessage(), e);
    }
    return ret;
}

Example 17 with HiveStorageHandler

use of org.apache.hadoop.hive.ql.metadata.HiveStorageHandler in project hive by apache.

the class PlanUtils method getTableDesc.

/**
 * Generate a table descriptor from a createViewDesc.
 */
public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, String colTypes) {
    TableDesc ret;
    try {
        HiveStorageHandler storageHandler = null;
        if (crtViewDesc.getStorageHandler() != null) {
            storageHandler = HiveUtils.getStorageHandler(SessionState.getSessionConf(), crtViewDesc.getStorageHandler());
        }
        Class<? extends Deserializer> serdeClass = getDefaultSerDe();
        String separatorCode = Integer.toString(Utilities.ctrlaCode);
        String columns = cols;
        String columnTypes = colTypes;
        boolean lastColumnTakesRestOfTheLine = false;
        if (storageHandler != null) {
            serdeClass = storageHandler.getSerDeClass();
        } else if (crtViewDesc.getSerde() != null) {
            serdeClass = JavaUtils.loadClass(crtViewDesc.getSerde());
        }
        ret = getTableDesc(serdeClass, separatorCode, columns, columnTypes, lastColumnTakesRestOfTheLine, false);
        // set other table properties
        Properties properties = ret.getProperties();
        if (crtViewDesc.getStorageHandler() != null) {
            properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, crtViewDesc.getStorageHandler());
        }
        if (crtViewDesc.getViewName() != null && crtViewDesc.isMaterialized()) {
            properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, crtViewDesc.getViewName());
        }
        if (crtViewDesc.getTblProps() != null) {
            properties.putAll(crtViewDesc.getTblProps());
        }
        if (crtViewDesc.getSerdeProps() != null) {
            properties.putAll(crtViewDesc.getSerdeProps());
        }
        // replace the default input & output file format with those found in
        // crtTblDesc
        Class<? extends InputFormat> in_class;
        if (storageHandler != null) {
            in_class = storageHandler.getInputFormatClass();
        } else {
            in_class = JavaUtils.loadClass(crtViewDesc.getInputFormat());
        }
        Class<? extends OutputFormat> out_class;
        if (storageHandler != null) {
            out_class = storageHandler.getOutputFormatClass();
        } else {
            out_class = JavaUtils.loadClass(crtViewDesc.getOutputFormat());
        }
        ret.setInputFileFormatClass(in_class);
        ret.setOutputFileFormatClass(out_class);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException("Unable to find class in getTableDesc: " + e.getMessage(), e);
    } catch (HiveException e) {
        throw new RuntimeException("Error loading storage handler in getTableDesc: " + e.getMessage(), e);
    }
    return ret;
}

Example 18 with HiveStorageHandler

use of org.apache.hadoop.hive.ql.metadata.HiveStorageHandler in project hive by apache.

the class OpProcFactory method pushFilterToStorageHandler.

/**
 * Attempts to push a predicate down into a storage handler.  For
 * native tables, this is a no-op.
 *
 * @param tableScanOp table scan against which predicate applies
 *
 * @param originalPredicate predicate to be pushed down
 *
 * @param owi object walk info
 *
 * @param hiveConf Hive configuration
 *
 * @return portion of predicate which needs to be evaluated
 * by Hive as a post-filter, or null if it was possible
 * to push down the entire predicate
 */
private static ExprNodeGenericFuncDesc pushFilterToStorageHandler(TableScanOperator tableScanOp, ExprNodeGenericFuncDesc originalPredicate, OpWalkerInfo owi, HiveConf hiveConf) throws SemanticException {
    TableScanDesc tableScanDesc = tableScanOp.getConf();
    Table tbl = tableScanDesc.getTableMetadata();
    if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER)) {
        // attach the original predicate to the table scan operator for index
        // optimizations that require the pushed predicate before pcr & later
        // optimizations are applied
        tableScanDesc.setFilterExpr(originalPredicate);
    }
    if (!tbl.isNonNative()) {
        return originalPredicate;
    }
    HiveStorageHandler storageHandler = tbl.getStorageHandler();
    if (!(storageHandler instanceof HiveStoragePredicateHandler)) {
        // The storage handler does not provide predicate decomposition
        // support, so we'll implement the entire filter in Hive.  However,
        // we still provide the full predicate to the storage handler in
        // case it wants to do any of its own prefiltering.
        tableScanDesc.setFilterExpr(originalPredicate);
        return originalPredicate;
    }
    HiveStoragePredicateHandler predicateHandler = (HiveStoragePredicateHandler) storageHandler;
    JobConf jobConf = new JobConf(owi.getParseContext().getConf());
    Utilities.setColumnNameList(jobConf, tableScanOp);
    Utilities.setColumnTypeList(jobConf, tableScanOp);
    try {
        Utilities.copyTableJobPropertiesToConf(Utilities.getTableDesc(tbl), jobConf);
    } catch (Exception e) {
        throw new SemanticException(e);
    }
    Deserializer deserializer = tbl.getDeserializer();
    HiveStoragePredicateHandler.DecomposedPredicate decomposed = predicateHandler.decomposePredicate(jobConf, deserializer, originalPredicate);
    if (decomposed == null) {
        // not able to push anything down
        if (LOG.isDebugEnabled()) {
            LOG.debug("No pushdown possible for predicate:  " + originalPredicate.getExprString());
        }
        return originalPredicate;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Original predicate:  " + originalPredicate.getExprString());
        if (decomposed.pushedPredicate != null) {
            LOG.debug("Pushed predicate:  " + decomposed.pushedPredicate.getExprString());
        }
        if (decomposed.residualPredicate != null) {
            LOG.debug("Residual predicate:  " + decomposed.residualPredicate.getExprString());
        }
    }
    tableScanDesc.setFilterExpr(decomposed.pushedPredicate);
    tableScanDesc.setFilterObject(decomposed.pushedPredicateObject);
    return decomposed.residualPredicate;
}