Examples with Partition org.apache.hadoop.hive.ql.metadata.Partition used on opensource projects

Example 46 with Partition

use of org.apache.hadoop.hive.ql.metadata.Partition in project hive by apache.

the class DDLTask method alterTableAlterPart.

/**
 * Alter partition column type in a table
 *
 * @param db
 *          Database to rename the partition.
 * @param alterPartitionDesc
 *          change partition column type.
 * @return Returns 0 when execution succeeds and above 0 if it fails.
 * @throws HiveException
 */
private int alterTableAlterPart(Hive db, AlterTableAlterPartDesc alterPartitionDesc) throws HiveException {
    Table tbl = db.getTable(alterPartitionDesc.getTableName(), true);
    // This is checked by DDLSemanticAnalyzer
    assert (tbl.isPartitioned());
    List<FieldSchema> newPartitionKeys = new ArrayList<FieldSchema>();
    // with a non null value before trying to alter the partition column type.
    try {
        Set<Partition> partitions = db.getAllPartitionsOf(tbl);
        int colIndex = -1;
        for (FieldSchema col : tbl.getTTable().getPartitionKeys()) {
            colIndex++;
            if (col.getName().compareTo(alterPartitionDesc.getPartKeySpec().getName()) == 0) {
                break;
            }
        }
        if (colIndex == -1 || colIndex == tbl.getTTable().getPartitionKeys().size()) {
            throw new HiveException("Cannot find partition column " + alterPartitionDesc.getPartKeySpec().getName());
        }
        TypeInfo expectedType = TypeInfoUtils.getTypeInfoFromTypeString(alterPartitionDesc.getPartKeySpec().getType());
        ObjectInspector outputOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
        Converter converter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
        // For all the existing partitions, check if the value can be type casted to a non-null object
        for (Partition part : partitions) {
            if (part.getName().equals(conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME))) {
                continue;
            }
            try {
                String value = part.getValues().get(colIndex);
                Object convertedValue = converter.convert(value);
                if (convertedValue == null) {
                    throw new HiveException(" Converting from " + TypeInfoFactory.stringTypeInfo + " to " + expectedType + " for value : " + value + " resulted in NULL object");
                }
            } catch (Exception e) {
                throw new HiveException("Exception while converting " + TypeInfoFactory.stringTypeInfo + " to " + expectedType + " for value : " + part.getValues().get(colIndex));
            }
        }
    } catch (Exception e) {
        throw new HiveException("Exception while checking type conversion of existing partition values to " + alterPartitionDesc.getPartKeySpec() + " : " + e.getMessage());
    }
    for (FieldSchema col : tbl.getTTable().getPartitionKeys()) {
        if (col.getName().compareTo(alterPartitionDesc.getPartKeySpec().getName()) == 0) {
            newPartitionKeys.add(alterPartitionDesc.getPartKeySpec());
        } else {
            newPartitionKeys.add(col);
        }
    }
    tbl.getTTable().setPartitionKeys(newPartitionKeys);
    db.alterTable(tbl, null);
    work.getInputs().add(new ReadEntity(tbl));
    // We've already locked the table as the input, don't relock it as the output.
    addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
    return 0;
}

Example 47 with Partition

use of org.apache.hadoop.hive.ql.metadata.Partition in project hive by apache.

the class DDLTask method touch.

/**
 * Rewrite the partition's metadata and force the pre/post execute hooks to
 * be fired.
 *
 * @param db
 * @param touchDesc
 * @return
 * @throws HiveException
 */
private int touch(Hive db, AlterTableSimpleDesc touchDesc) throws HiveException {
    Table tbl = db.getTable(touchDesc.getTableName());
    EnvironmentContext environmentContext = new EnvironmentContext();
    environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
    if (touchDesc.getPartSpec() == null) {
        db.alterTable(tbl, environmentContext);
        work.getInputs().add(new ReadEntity(tbl));
        addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
    } else {
        Partition part = db.getPartition(tbl, touchDesc.getPartSpec(), false);
        if (part == null) {
            throw new HiveException("Specified partition does not exist");
        }
        try {
            db.alterPartition(touchDesc.getTableName(), part, environmentContext);
        } catch (InvalidOperationException e) {
            throw new HiveException(e);
        }
        work.getInputs().add(new ReadEntity(part));
        addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK));
    }
    return 0;
}

Example 48 with Partition

use of org.apache.hadoop.hive.ql.metadata.Partition in project hive by apache.

the class DDLTask method exchangeTablePartition.

private int exchangeTablePartition(Hive db, AlterTableExchangePartition exchangePartition) throws HiveException {
    Map<String, String> partitionSpecs = exchangePartition.getPartitionSpecs();
    Table destTable = exchangePartition.getDestinationTable();
    Table sourceTable = exchangePartition.getSourceTable();
    List<Partition> partitions = db.exchangeTablePartitions(partitionSpecs, sourceTable.getDbName(), sourceTable.getTableName(), destTable.getDbName(), destTable.getTableName());
    for (Partition partition : partitions) {
        // Reuse the partition specs from dest partition since they should be the same
        work.getInputs().add(new ReadEntity(new Partition(sourceTable, partition.getSpec(), null)));
        addIfAbsentByName(new WriteEntity(new Partition(sourceTable, partition.getSpec(), null), WriteEntity.WriteType.DELETE));
        addIfAbsentByName(new WriteEntity(new Partition(destTable, partition.getSpec(), null), WriteEntity.WriteType.INSERT));
    }
    return 0;
}

Example 49 with Partition

use of org.apache.hadoop.hive.ql.metadata.Partition in project hive by apache.

the class MoveTask method checkFileFormats.

private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table) throws HiveException {
    if (work.getCheckFileFormat()) {
        // Get all files from the src directory
        FileStatus[] dirs;
        ArrayList<FileStatus> files;
        // source filesystem
        FileSystem srcFs;
        try {
            srcFs = tbd.getSourcePath().getFileSystem(conf);
            dirs = srcFs.globStatus(tbd.getSourcePath());
            files = new ArrayList<FileStatus>();
            for (int i = 0; (dirs != null && i < dirs.length); i++) {
                files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER)));
                // one.
                if (files.size() > 0) {
                    break;
                }
            }
        } catch (IOException e) {
            throw new HiveException("addFiles: filesystem error in check phase", e);
        }
        // handle file format check for table level
        if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
            boolean flag = true;
            // dynamic partition context is null
            if (tbd.getDPCtx() == null) {
                if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) {
                    // Check if the file format of the file matches that of the table.
                    flag = HiveFileFormatUtils.checkInputFormat(srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
                } else {
                    // Check if the file format of the file matches that of the partition
                    Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false);
                    if (oldPart == null) {
                        // this means we have just created a table and are specifying partition in the
                        // load statement (without pre-creating the partition), in which case lets use
                        // table input format class. inheritTableSpecs defaults to true so when a new
                        // partition is created later it will automatically inherit input format
                        // from table object
                        flag = HiveFileFormatUtils.checkInputFormat(srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
                    } else {
                        flag = HiveFileFormatUtils.checkInputFormat(srcFs, conf, oldPart.getInputFormatClass(), files);
                    }
                }
                if (!flag) {
                    throw new HiveException(ErrorMsg.WRONG_FILE_FORMAT);
                }
            } else {
                LOG.warn("Skipping file format check as dpCtx is not null");
            }
        }
    }
}

Example 50 with Partition

use of org.apache.hadoop.hive.ql.metadata.Partition in project hive by apache.

the class MoveTask method handleStaticParts.

private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
    List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec());
    db.validatePartitionNameCharacters(partVals);
    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
        Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() + " into " + tbd.getTable().getTableName());
    }
    db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getLoadFileType(), tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && !tbd.isMmTable(), hasFollowingStatsTask(), tbd.getWriteId(), tbd.getStmtId());
    Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
    // See the comment inside updatePartitionBucketSortColumns.
    if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
        updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols);
    }
    DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition());
    // add this partition to post-execution hook
    if (work.getOutputs() != null) {
        DDLTask.addIfAbsentByName(new WriteEntity(partn, getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
    }
    return dc;
}