Examples with ReadEntity org.apache.hadoop.hive.ql.hooks.ReadEntity used on opensource projects

Example 1 with ReadEntity

use of org.apache.hadoop.hive.ql.hooks.ReadEntity in project hive by apache.

the class DDLSemanticAnalyzer method analyzeCreateIndex.

private void analyzeCreateIndex(ASTNode ast) throws SemanticException {
    String indexName = unescapeIdentifier(ast.getChild(0).getText());
    String typeName = unescapeSQLString(ast.getChild(1).getText());
    String[] qTabName = getQualifiedTableName((ASTNode) ast.getChild(2));
    List<String> indexedCols = getColumnNames((ASTNode) ast.getChild(3));
    IndexType indexType = HiveIndex.getIndexType(typeName);
    if (indexType != null) {
        typeName = indexType.getHandlerClsName();
    } else {
        try {
            JavaUtils.loadClass(typeName);
        } catch (Exception e) {
            throw new SemanticException("class name provided for index handler not found.", e);
        }
    }
    String indexTableName = null;
    boolean deferredRebuild = false;
    String location = null;
    Map<String, String> tblProps = null;
    Map<String, String> idxProps = null;
    String indexComment = null;
    RowFormatParams rowFormatParams = new RowFormatParams();
    StorageFormat storageFormat = new StorageFormat(conf);
    for (int idx = 4; idx < ast.getChildCount(); idx++) {
        ASTNode child = (ASTNode) ast.getChild(idx);
        if (storageFormat.fillStorageFormat(child)) {
            continue;
        }
        switch(child.getToken().getType()) {
            case HiveParser.TOK_TABLEROWFORMAT:
                rowFormatParams.analyzeRowFormat(child);
                break;
            case HiveParser.TOK_CREATEINDEX_INDEXTBLNAME:
                ASTNode ch = (ASTNode) child.getChild(0);
                indexTableName = getUnescapedName(ch);
                break;
            case HiveParser.TOK_DEFERRED_REBUILDINDEX:
                deferredRebuild = true;
                break;
            case HiveParser.TOK_TABLELOCATION:
                location = unescapeSQLString(child.getChild(0).getText());
                addLocationToOutputs(location);
                break;
            case HiveParser.TOK_TABLEPROPERTIES:
                tblProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0));
                break;
            case HiveParser.TOK_INDEXPROPERTIES:
                idxProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0));
                break;
            case HiveParser.TOK_TABLESERIALIZER:
                child = (ASTNode) child.getChild(0);
                storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText()));
                if (child.getChildCount() == 2) {
                    readProps((ASTNode) (child.getChild(1).getChild(0)), storageFormat.getSerdeProps());
                }
                break;
            case HiveParser.TOK_INDEXCOMMENT:
                child = (ASTNode) child.getChild(0);
                indexComment = unescapeSQLString(child.getText());
        }
    }
    storageFormat.fillDefaultStorageFormat(false, false);
    if (indexTableName == null) {
        indexTableName = MetaStoreUtils.getIndexTableName(qTabName[0], qTabName[1], indexName);
        // on same database with base table
        indexTableName = qTabName[0] + "." + indexTableName;
    } else {
        indexTableName = getDotName(Utilities.getDbTableName(indexTableName));
    }
    inputs.add(new ReadEntity(getTable(qTabName)));
    CreateIndexDesc crtIndexDesc = new CreateIndexDesc(getDotName(qTabName), indexName, indexedCols, indexTableName, deferredRebuild, storageFormat.getInputFormat(), storageFormat.getOutputFormat(), storageFormat.getStorageHandler(), typeName, location, idxProps, tblProps, storageFormat.getSerde(), storageFormat.getSerdeProps(), rowFormatParams.collItemDelim, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.lineDelim, rowFormatParams.mapKeyDelim, indexComment);
    Task<?> createIndex = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), crtIndexDesc), conf);
    rootTasks.add(createIndex);
}

Example 2 with ReadEntity

use of org.apache.hadoop.hive.ql.hooks.ReadEntity in project hive by apache.

the class DDLSemanticAnalyzer method addInputsOutputsAlterTable.

private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec, AlterTableDesc desc, AlterTableTypes op) throws SemanticException {
    boolean isCascade = desc != null && desc.getIsCascade();
    boolean alterPartitions = partSpec != null && !partSpec.isEmpty();
    //cascade only occurs at table level then cascade to partition level
    if (isCascade && alterPartitions) {
        throw new SemanticException(ErrorMsg.ALTER_TABLE_PARTITION_CASCADE_NOT_SUPPORTED, op.getName());
    }
    Table tab = getTable(tableName, true);
    // Determine the lock type to acquire
    WriteEntity.WriteType writeType = WriteEntity.determineAlterTableWriteType(op);
    if (!alterPartitions) {
        inputs.add(new ReadEntity(tab));
        outputs.add(new WriteEntity(tab, writeType));
        //do not need the lock for partitions since they are covered by the table lock
        if (isCascade) {
            for (Partition part : getPartitions(tab, partSpec, false)) {
                outputs.add(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK));
            }
        }
    } else {
        ReadEntity re = new ReadEntity(tab);
        // In the case of altering a table for its partitions we don't need to lock the table
        // itself, just the partitions.  But the table will have a ReadEntity.  So mark that
        // ReadEntity as no lock.
        re.noLockNeeded();
        inputs.add(re);
        if (isFullSpec(tab, partSpec)) {
            // Fully specified partition spec
            Partition part = getPartition(tab, partSpec, true);
            outputs.add(new WriteEntity(part, writeType));
        } else {
            // Partial partition spec supplied. Make sure this is allowed.
            if (!AlterTableDesc.doesAlterTableTypeSupportPartialPartitionSpec(op)) {
                throw new SemanticException(ErrorMsg.ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED, op.getName());
            } else if (!conf.getBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING)) {
                throw new SemanticException(ErrorMsg.DYNAMIC_PARTITION_DISABLED);
            }
            for (Partition part : getPartitions(tab, partSpec, true)) {
                outputs.add(new WriteEntity(part, writeType));
            }
        }
    }
    if (desc != null) {
        validateAlterTableType(tab, op, desc.getExpectView());
        // validate Unset Non Existed Table Properties
        if (op == AlterTableDesc.AlterTableTypes.DROPPROPS && !desc.getIsDropIfExists()) {
            Map<String, String> tableParams = tab.getTTable().getParameters();
            for (String currKey : desc.getProps().keySet()) {
                if (!tableParams.containsKey(currKey)) {
                    String errorMsg = "The following property " + currKey + " does not exist in " + tab.getTableName();
                    throw new SemanticException(ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg));
                }
            }
        }
    }
}

Example 3 with ReadEntity

use of org.apache.hadoop.hive.ql.hooks.ReadEntity in project hive by apache.

the class DDLTask method alterTable.

/**
 * Alter a given table.
 *
 * @param db
 *          The database in question.
 * @param alterTbl
 *          This is the table we're altering.
 * @return Returns 0 when execution succeeds and above 0 if it fails.
 * @throws HiveException
 *           Throws this exception if an unexpected error occurs.
 */
private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException {
    if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
        String[] names = Utilities.getDbTableName(alterTbl.getOldName());
        if (Utils.isBootstrapDumpInProgress(db, names[0])) {
            LOG.error("DDLTask: Rename Table not allowed as bootstrap dump in progress");
            throw new HiveException("Rename Table: Not allowed as bootstrap dump in progress");
        }
    }
    // alter the table
    Table tbl = db.getTable(alterTbl.getOldName());
    List<Partition> allPartitions = null;
    if (alterTbl.getPartSpec() != null) {
        Map<String, String> partSpec = alterTbl.getPartSpec();
        if (DDLSemanticAnalyzer.isFullSpec(tbl, partSpec)) {
            allPartitions = new ArrayList<Partition>();
            Partition part = db.getPartition(tbl, partSpec, false);
            if (part == null) {
                // User provided a fully specified partition spec but it doesn't exist, fail.
                throw new HiveException(ErrorMsg.INVALID_PARTITION, StringUtils.join(alterTbl.getPartSpec().keySet(), ',') + " for table " + alterTbl.getOldName());
            }
            allPartitions.add(part);
        } else {
            // DDLSemanticAnalyzer has already checked if partial partition specs are allowed,
            // thus we should not need to check it here.
            allPartitions = db.getPartitions(tbl, alterTbl.getPartSpec());
        }
    }
    // Don't change the table object returned by the metastore, as we'll mess with it's caches.
    Table oldTbl = tbl;
    tbl = oldTbl.copy();
    // but let's make it a little bit more explicit.
    if (allPartitions != null) {
        // Alter all partitions
        for (Partition part : allPartitions) {
            addChildTasks(alterTableOrSinglePartition(alterTbl, tbl, part));
        }
    } else {
        // Just alter the table
        addChildTasks(alterTableOrSinglePartition(alterTbl, tbl, null));
    }
    if (allPartitions == null) {
        updateModifiedParameters(tbl.getTTable().getParameters(), conf);
        tbl.checkValidity(conf);
    } else {
        for (Partition tmpPart : allPartitions) {
            updateModifiedParameters(tmpPart.getParameters(), conf);
        }
    }
    try {
        if (allPartitions == null) {
            db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), alterTbl.getEnvironmentContext());
        } else {
            db.alterPartitions(Warehouse.getQualifiedName(tbl.getTTable()), allPartitions, alterTbl.getEnvironmentContext());
        }
        // Add constraints if necessary
        addConstraints(db, alterTbl);
    } catch (InvalidOperationException e) {
        LOG.error("alter table: ", e);
        throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
    }
    // Don't acquire locks for any of these, we have already asked for them in DDLSemanticAnalyzer.
    if (allPartitions != null) {
        for (Partition tmpPart : allPartitions) {
            work.getInputs().add(new ReadEntity(tmpPart));
            addIfAbsentByName(new WriteEntity(tmpPart, WriteEntity.WriteType.DDL_NO_LOCK));
        }
    } else {
        work.getInputs().add(new ReadEntity(oldTbl));
        addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
    }
    return 0;
}

Example 4 with ReadEntity

use of org.apache.hadoop.hive.ql.hooks.ReadEntity in project hive by apache.

the class DDLTask method alterTableAlterPart.

/**
 * Alter partition column type in a table
 *
 * @param db
 *          Database to rename the partition.
 * @param alterPartitionDesc
 *          change partition column type.
 * @return Returns 0 when execution succeeds and above 0 if it fails.
 * @throws HiveException
 */
private int alterTableAlterPart(Hive db, AlterTableAlterPartDesc alterPartitionDesc) throws HiveException {
    Table tbl = db.getTable(alterPartitionDesc.getTableName(), true);
    // This is checked by DDLSemanticAnalyzer
    assert (tbl.isPartitioned());
    List<FieldSchema> newPartitionKeys = new ArrayList<FieldSchema>();
    // with a non null value before trying to alter the partition column type.
    try {
        Set<Partition> partitions = db.getAllPartitionsOf(tbl);
        int colIndex = -1;
        for (FieldSchema col : tbl.getTTable().getPartitionKeys()) {
            colIndex++;
            if (col.getName().compareTo(alterPartitionDesc.getPartKeySpec().getName()) == 0) {
                break;
            }
        }
        if (colIndex == -1 || colIndex == tbl.getTTable().getPartitionKeys().size()) {
            throw new HiveException("Cannot find partition column " + alterPartitionDesc.getPartKeySpec().getName());
        }
        TypeInfo expectedType = TypeInfoUtils.getTypeInfoFromTypeString(alterPartitionDesc.getPartKeySpec().getType());
        ObjectInspector outputOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
        Converter converter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
        // For all the existing partitions, check if the value can be type casted to a non-null object
        for (Partition part : partitions) {
            if (part.getName().equals(conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME))) {
                continue;
            }
            try {
                String value = part.getValues().get(colIndex);
                Object convertedValue = converter.convert(value);
                if (convertedValue == null) {
                    throw new HiveException(" Converting from " + TypeInfoFactory.stringTypeInfo + " to " + expectedType + " for value : " + value + " resulted in NULL object");
                }
            } catch (Exception e) {
                throw new HiveException("Exception while converting " + TypeInfoFactory.stringTypeInfo + " to " + expectedType + " for value : " + part.getValues().get(colIndex));
            }
        }
    } catch (Exception e) {
        throw new HiveException("Exception while checking type conversion of existing partition values to " + alterPartitionDesc.getPartKeySpec() + " : " + e.getMessage());
    }
    for (FieldSchema col : tbl.getTTable().getPartitionKeys()) {
        if (col.getName().compareTo(alterPartitionDesc.getPartKeySpec().getName()) == 0) {
            newPartitionKeys.add(alterPartitionDesc.getPartKeySpec());
        } else {
            newPartitionKeys.add(col);
        }
    }
    tbl.getTTable().setPartitionKeys(newPartitionKeys);
    db.alterTable(tbl, null);
    work.getInputs().add(new ReadEntity(tbl));
    // We've already locked the table as the input, don't relock it as the output.
    addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
    return 0;
}

Example 5 with ReadEntity

use of org.apache.hadoop.hive.ql.hooks.ReadEntity in project hive by apache.

the class DDLTask method touch.

/**
 * Rewrite the partition's metadata and force the pre/post execute hooks to
 * be fired.
 *
 * @param db
 * @param touchDesc
 * @return
 * @throws HiveException
 */
private int touch(Hive db, AlterTableSimpleDesc touchDesc) throws HiveException {
    Table tbl = db.getTable(touchDesc.getTableName());
    EnvironmentContext environmentContext = new EnvironmentContext();
    environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
    if (touchDesc.getPartSpec() == null) {
        db.alterTable(tbl, environmentContext);
        work.getInputs().add(new ReadEntity(tbl));
        addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
    } else {
        Partition part = db.getPartition(tbl, touchDesc.getPartSpec(), false);
        if (part == null) {
            throw new HiveException("Specified partition does not exist");
        }
        try {
            db.alterPartition(touchDesc.getTableName(), part, environmentContext);
        } catch (InvalidOperationException e) {
            throw new HiveException(e);
        }
        work.getInputs().add(new ReadEntity(part));
        addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK));
    }
    return 0;
}