Examples with HiveException org.apache.hadoop.hive.ql.metadata.HiveException used on opensource projects

Example 81 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class SemanticAnalyzer method walkASTMarkTABREF.

private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set<String> cteAlias, Context ctx, Hive db, Map<String, Table> tabNameToTabObject, Set<Integer> ignoredTokens) throws SemanticException {
    Queue<Node> queue = new LinkedList<>();
    queue.add(ast);
    Map<HivePrivilegeObject, MaskAndFilterInfo> basicInfos = new LinkedHashMap<>();
    while (!queue.isEmpty()) {
        ASTNode astNode = (ASTNode) queue.poll();
        if (astNode.getToken().getType() == HiveParser.TOK_TABREF) {
            int aliasIndex = 0;
            StringBuilder additionalTabInfo = new StringBuilder();
            for (int index = 1; index < astNode.getChildCount(); index++) {
                ASTNode ct = (ASTNode) astNode.getChild(index);
                if (ct.getToken().getType() == HiveParser.TOK_TABLEBUCKETSAMPLE || ct.getToken().getType() == HiveParser.TOK_TABLESPLITSAMPLE || ct.getToken().getType() == HiveParser.TOK_TABLEPROPERTIES) {
                    additionalTabInfo.append(ctx.getTokenRewriteStream().toString(ct.getTokenStartIndex(), ct.getTokenStopIndex()));
                } else {
                    aliasIndex = index;
                }
            }
            ASTNode tableTree = (ASTNode) (astNode.getChild(0));
            String tabIdName = getUnescapedName(tableTree);
            String alias;
            if (aliasIndex != 0) {
                alias = unescapeIdentifier(astNode.getChild(aliasIndex).getText());
            } else {
                alias = getUnescapedUnqualifiedTableName(tableTree);
            }
            // select * from TAB2 [no masking]
            if (cteAlias.contains(tabIdName)) {
                continue;
            }
            String replacementText = null;
            Table table = null;
            try {
                if (!tabNameToTabObject.containsKey(tabIdName)) {
                    table = db.getTable(tabIdName, true);
                    tabNameToTabObject.put(tabIdName, table);
                } else {
                    table = tabNameToTabObject.get(tabIdName);
                }
            } catch (HiveException e) {
                // Table may not be found when materialization of CTE is on.
                STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF.");
                continue;
            }
            List<String> colNames = new ArrayList<>();
            List<String> colTypes = new ArrayList<>();
            for (FieldSchema col : table.getAllCols()) {
                colNames.add(col.getName());
                colTypes.add(col.getType());
            }
            basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView()));
        }
        if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) {
            for (Node child : astNode.getChildren()) {
                queue.offer(child);
            }
        }
    }
    List<HivePrivilegeObject> basicPrivObjs = new ArrayList<>();
    basicPrivObjs.addAll(basicInfos.keySet());
    List<HivePrivilegeObject> needRewritePrivObjs = tableMask.applyRowFilterAndColumnMasking(basicPrivObjs);
    if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) {
        for (HivePrivilegeObject privObj : needRewritePrivObjs) {
            MaskAndFilterInfo info = basicInfos.get(privObj);
            String replacementText = tableMask.create(privObj, info);
            if (replacementText != null) {
                // We don't support masking/filtering against ACID query at the moment
                if (ctx.getIsUpdateDeleteMerge()) {
                    throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_ACID_NOT_SUPPORTED, privObj.getDbname(), privObj.getObjectName());
                }
                tableMask.setNeedsRewrite(true);
                tableMask.addTranslation(info.astNode, replacementText);
            }
        }
    }
}

Example 82 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class SemanticAnalyzer method getStagingDirectoryPathname.

/**
 * Gets the staging directory where MR files will be stored temporary.
 * It walks through the QB plan to find the correct location where save temporary files. This
 * temporary location (or staging directory) may be created inside encrypted tables locations for
 * security reasons. If the QB has read-only tables, then the older scratch directory will be used,
 * or a permission error will be thrown if the requested query table is encrypted and the old scratch
 * directory is not.
 *
 * @param qb The QB object that contains a list of all table locations.
 * @return The path to the staging directory.
 * @throws HiveException If an error occurs while identifying the correct staging location.
 */
private Path getStagingDirectoryPathname(QB qb) throws HiveException {
    Path stagingPath = null, tablePath;
    // Looks for the most encrypted table location
    // It may return null if there are not tables encrypted, or are not part of HDFS
    tablePath = getStrongestEncryptedTablePath(qb);
    if (tablePath != null) {
        // At this point, tablePath is part of HDFS and it is encrypted
        if (isPathReadOnly(tablePath)) {
            Path tmpPath = ctx.getMRTmpPath();
            if (comparePathKeyStrength(tablePath, tmpPath) < 0) {
                throw new HiveException("Read-only encrypted tables cannot be read " + "if the scratch directory is not encrypted (or encryption is weak)");
            } else {
                stagingPath = tmpPath;
            }
        }
        if (stagingPath == null) {
            stagingPath = ctx.getMRTmpPath(tablePath.toUri());
        }
    } else {
        stagingPath = ctx.getMRTmpPath();
    }
    return stagingPath;
}

Example 83 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class BaseSemanticAnalyzer method isValidPrefixSpec.

/**
 * Checks if given specification is proper specification for prefix of
 * partition cols, for table partitioned by ds, hr, min valid ones are
 * (ds='2008-04-08'), (ds='2008-04-08', hr='12'), (ds='2008-04-08', hr='12', min='30')
 * invalid one is for example (ds='2008-04-08', min='30')
 * @param spec specification key-value map
 * @return true if the specification is prefix; never returns false, but throws
 * @throws HiveException
 */
public final boolean isValidPrefixSpec(Table tTable, Map<String, String> spec) throws HiveException {
    // TODO - types need to be checked.
    List<FieldSchema> partCols = tTable.getPartitionKeys();
    if (partCols == null || (partCols.size() == 0)) {
        if (spec != null) {
            throw new HiveException("table is not partitioned but partition spec exists: " + spec);
        } else {
            return true;
        }
    }
    if (spec == null) {
        throw new HiveException("partition spec is not specified");
    }
    Iterator<String> itrPsKeys = spec.keySet().iterator();
    for (FieldSchema fs : partCols) {
        if (!itrPsKeys.hasNext()) {
            break;
        }
        if (!itrPsKeys.next().toLowerCase().equals(fs.getName().toLowerCase())) {
            ErrorPartSpec(spec, partCols);
        }
    }
    if (itrPsKeys.hasNext()) {
        ErrorPartSpec(spec, partCols);
    }
    return true;
}

Example 84 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class DDLSemanticAnalyzer method analyzeAlterTableArchive.

private void analyzeAlterTableArchive(String[] qualified, CommonTree ast, boolean isUnArchive) throws SemanticException {
    if (!conf.getBoolVar(HiveConf.ConfVars.HIVEARCHIVEENABLED)) {
        throw new SemanticException(ErrorMsg.ARCHIVE_METHODS_DISABLED.getMsg());
    }
    Table tab = getTable(qualified);
    // partition name to value
    List<Map<String, String>> partSpecs = getPartitionSpecs(tab, ast);
    addTablePartsOutputs(tab, partSpecs, true, WriteEntity.WriteType.DDL_NO_LOCK);
    validateAlterTableType(tab, AlterTableTypes.ARCHIVE);
    inputs.add(new ReadEntity(tab));
    if (partSpecs.size() > 1) {
        throw new SemanticException(isUnArchive ? ErrorMsg.UNARCHIVE_ON_MULI_PARTS.getMsg() : ErrorMsg.ARCHIVE_ON_MULI_PARTS.getMsg());
    }
    if (partSpecs.size() == 0) {
        throw new SemanticException(ErrorMsg.ARCHIVE_ON_TABLE.getMsg());
    }
    Map<String, String> partSpec = partSpecs.get(0);
    try {
        isValidPrefixSpec(tab, partSpec);
    } catch (HiveException e) {
        throw new SemanticException(e.getMessage(), e);
    }
    AlterTableSimpleDesc archiveDesc = new AlterTableSimpleDesc(getDotName(qualified), partSpec, (isUnArchive ? AlterTableTypes.UNARCHIVE : AlterTableTypes.ARCHIVE));
    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), archiveDesc)));
}

Example 85 with HiveException

use of org.apache.hadoop.hive.ql.metadata.HiveException in project hive by apache.

the class DDLSemanticAnalyzer method analyzeAlterTablePartMergeFiles.

private void analyzeAlterTablePartMergeFiles(ASTNode ast, String tableName, HashMap<String, String> partSpec) throws SemanticException {
    AlterTablePartMergeFilesDesc mergeDesc = new AlterTablePartMergeFilesDesc(tableName, partSpec);
    List<Path> inputDir = new ArrayList<Path>();
    Path oldTblPartLoc = null;
    Path newTblPartLoc = null;
    Table tblObj = null;
    ListBucketingCtx lbCtx = null;
    try {
        tblObj = getTable(tableName);
        // TODO: we should probably block all ACID tables here.
        if (AcidUtils.isInsertOnlyTable(tblObj.getParameters())) {
            throw new SemanticException("Merge is not supported for MM tables");
        }
        mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj));
        List<String> bucketCols = null;
        Class<? extends InputFormat> inputFormatClass = null;
        boolean isArchived = false;
        if (tblObj.isPartitioned()) {
            if (partSpec == null) {
                throw new SemanticException("source table " + tableName + " is partitioned but no partition desc found.");
            } else {
                Partition part = getPartition(tblObj, partSpec, false);
                if (part == null) {
                    throw new SemanticException("source table " + tableName + " is partitioned but partition not found.");
                }
                bucketCols = part.getBucketCols();
                inputFormatClass = part.getInputFormatClass();
                isArchived = ArchiveUtils.isArchived(part);
                Path tabPath = tblObj.getPath();
                Path partPath = part.getDataLocation();
                // if the table is in a different dfs than the partition,
                // replace the partition's dfs with the table's dfs.
                newTblPartLoc = new Path(tabPath.toUri().getScheme(), tabPath.toUri().getAuthority(), partPath.toUri().getPath());
                oldTblPartLoc = partPath;
                lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(), part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf);
            }
        } else {
            inputFormatClass = tblObj.getInputFormatClass();
            bucketCols = tblObj.getBucketCols();
            // input and output are the same
            oldTblPartLoc = tblObj.getPath();
            newTblPartLoc = tblObj.getPath();
            lbCtx = constructListBucketingCtx(tblObj.getSkewedColNames(), tblObj.getSkewedColValues(), tblObj.getSkewedColValueLocationMaps(), tblObj.isStoredAsSubDirectories(), conf);
        }
        // throw a HiveException for other than rcfile and orcfile.
        if (!((inputFormatClass.equals(RCFileInputFormat.class) || (inputFormatClass.equals(OrcInputFormat.class))))) {
            throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_FILE_FORMAT.getMsg());
        }
        mergeDesc.setInputFormatClass(inputFormatClass);
        // throw a HiveException if the table/partition is bucketized
        if (bucketCols != null && bucketCols.size() > 0) {
            throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_BUCKETED.getMsg());
        }
        // throw a HiveException if the table/partition is archived
        if (isArchived) {
            throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_PARTITION_ARCHIVED.getMsg());
        }
        // violating which can cause data loss
        if (tblObj.isNonNative()) {
            throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NON_NATIVE.getMsg());
        }
        if (tblObj.getTableType() != TableType.MANAGED_TABLE) {
            throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NOT_MANAGED.getMsg());
        }
        // transactional tables are compacted and no longer needs to be bucketed, so not safe for merge/concatenation
        boolean isAcid = AcidUtils.isTransactionalTable(tblObj);
        if (isAcid) {
            throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_TRANSACTIONAL.getMsg());
        }
        inputDir.add(oldTblPartLoc);
        mergeDesc.setInputDir(inputDir);
        mergeDesc.setLbCtx(lbCtx);
        addInputsOutputsAlterTable(tableName, partSpec, AlterTableTypes.MERGEFILES);
        DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc);
        ddlWork.setNeedLock(true);
        Task<? extends Serializable> mergeTask = TaskFactory.get(ddlWork);
        TableDesc tblDesc = Utilities.getTableDesc(tblObj);
        Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc);
        mergeDesc.setOutputDir(queryTmpdir);
        // No need to handle MM tables - unsupported path.
        LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc, partSpec == null ? new HashMap<>() : partSpec);
        ltd.setLbCtx(lbCtx);
        Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false));
        mergeTask.addDependentTask(moveTsk);
        if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
            BasicStatsWork basicStatsWork;
            if (oldTblPartLoc.equals(newTblPartLoc)) {
                // If we're merging to the same location, we can avoid some metastore calls
                TableSpec tableSpec = new TableSpec(db, tableName, partSpec);
                basicStatsWork = new BasicStatsWork(tableSpec);
            } else {
                basicStatsWork = new BasicStatsWork(ltd);
            }
            basicStatsWork.setNoStatsAggregator(true);
            basicStatsWork.setClearAggregatorStats(true);
            StatsWork columnStatsWork = new StatsWork(tblObj, basicStatsWork, conf);
            Task<? extends Serializable> statTask = TaskFactory.get(columnStatsWork);
            moveTsk.addDependentTask(statTask);
        }
        rootTasks.add(mergeTask);
    } catch (Exception e) {
        throw new SemanticException(e);
    }
}