Examples with InvalidObjectException org.apache.hadoop.hive.metastore.api.InvalidObjectException used on opensource projects

Example 41 with InvalidObjectException

use of org.apache.hadoop.hive.metastore.api.InvalidObjectException in project hive by apache.

the class HiveAlterHandler method updateOrGetPartitionColumnStats.

private ColumnStatistics updateOrGetPartitionColumnStats(RawStore msdb, String dbname, String tblname, List<String> partVals, List<FieldSchema> oldCols, Table table, Partition part, List<FieldSchema> newCols) throws MetaException, InvalidObjectException {
    ColumnStatistics newPartsColStats = null;
    try {
        // if newCols are not specified, use default ones.
        if (newCols == null) {
            newCols = part.getSd() == null ? new ArrayList<>() : part.getSd().getCols();
        }
        String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals);
        String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
        boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname) || !oldPartName.equals(newPartName);
        // do not need to update column stats if alter partition is not for rename or changing existing columns
        if (!rename && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) {
            return newPartsColStats;
        }
        List<String> oldColNames = new ArrayList<>(oldCols.size());
        for (FieldSchema oldCol : oldCols) {
            oldColNames.add(oldCol.getName());
        }
        List<String> oldPartNames = Lists.newArrayList(oldPartName);
        List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbname, tblname, oldPartNames, oldColNames);
        assert (partsColStats.size() <= 1);
        for (ColumnStatistics partColStats : partsColStats) {
            // actually only at most one loop
            List<ColumnStatisticsObj> newStatsObjs = new ArrayList<>();
            List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj();
            List<String> deletedCols = new ArrayList<>();
            for (ColumnStatisticsObj statsObj : statsObjs) {
                boolean found = false;
                for (FieldSchema newCol : newCols) {
                    if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
                        found = true;
                        break;
                    }
                }
                if (found) {
                    if (rename) {
                        msdb.deletePartitionColumnStatistics(dbname, tblname, partColStats.getStatsDesc().getPartName(), partVals, statsObj.getColName());
                        newStatsObjs.add(statsObj);
                    }
                } else {
                    msdb.deletePartitionColumnStatistics(dbname, tblname, partColStats.getStatsDesc().getPartName(), partVals, statsObj.getColName());
                    deletedCols.add(statsObj.getColName());
                }
            }
            StatsSetupConst.removeColumnStatsState(part.getParameters(), deletedCols);
            if (!newStatsObjs.isEmpty()) {
                partColStats.setStatsObj(newStatsObjs);
                newPartsColStats = partColStats;
            }
        }
    } catch (NoSuchObjectException nsoe) {
    // ignore this exception, actually this exception won't be thrown from getPartitionColumnStatistics
    } catch (InvalidInputException iie) {
        throw new InvalidObjectException("Invalid input to delete partition column stats." + iie);
    }
    return newPartsColStats;
}

Example 42 with InvalidObjectException

use of org.apache.hadoop.hive.metastore.api.InvalidObjectException in project hive by apache.

the class HiveAlterHandler method alterTable.

@Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, IHMSHandler handler) throws InvalidOperationException, MetaException {
    name = name.toLowerCase();
    dbname = dbname.toLowerCase();
    final boolean cascade = environmentContext != null && environmentContext.isSetProperties() && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(StatsSetupConst.CASCADE));
    if (newt == null) {
        throw new InvalidOperationException("New table is invalid: " + newt);
    }
    String newTblName = newt.getTableName().toLowerCase();
    String newDbName = newt.getDbName().toLowerCase();
    if (!MetaStoreUtils.validateName(newTblName, handler.getConf())) {
        throw new InvalidOperationException(newTblName + " is not a valid object name");
    }
    String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols());
    if (validate != null) {
        throw new InvalidOperationException("Invalid column " + validate);
    }
    Path srcPath = null;
    FileSystem srcFs;
    Path destPath = null;
    FileSystem destFs = null;
    boolean success = false;
    boolean dataWasMoved = false;
    Table oldt;
    List<TransactionalMetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    try {
        boolean rename = false;
        boolean isPartitionedTable = false;
        List<Partition> parts;
        // check if table with the new name already exists
        if (!newTblName.equals(name) || !newDbName.equals(dbname)) {
            if (msdb.getTable(newDbName, newTblName) != null) {
                throw new InvalidOperationException("new table " + newDbName + "." + newTblName + " already exists");
            }
            rename = true;
        }
        msdb.openTransaction();
        // get old table
        oldt = msdb.getTable(dbname, name);
        if (oldt == null) {
            throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
        }
        if (oldt.getPartitionKeysSize() != 0) {
            isPartitionedTable = true;
        }
        // be done only for non-views.
        if (MetastoreConf.getBoolVar(handler.getConf(), MetastoreConf.ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES) && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) {
            // Throws InvalidOperationException if the new column types are not
            // compatible with the current column types.
            checkColTypeChangeCompatible(oldt.getSd().getCols(), newt.getSd().getCols());
        }
        // check that partition keys have not changed, except for virtual views
        // however, allow the partition comments to change
        boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(), newt.getPartitionKeys());
        if (!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) {
            if (!partKeysPartiallyEqual) {
                throw new InvalidOperationException("partition keys can not be changed.");
            }
        }
        // 4) the table was not initially created with a specified location
        if (rename && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 || StringUtils.isEmpty(newt.getSd().getLocation())) && !MetaStoreUtils.isExternalTable(oldt)) {
            Database olddb = msdb.getDatabase(dbname);
            // if a table was created in a user specified location using the DDL like
            // create table tbl ... location ...., it should be treated like an external table
            // in the table rename, its data location should not be changed. We can check
            // if the table directory was created directly under its database directory to tell
            // if it is such a table
            srcPath = new Path(oldt.getSd().getLocation());
            String oldtRelativePath = (new Path(olddb.getLocationUri()).toUri()).relativize(srcPath.toUri()).toString();
            boolean tableInSpecifiedLoc = !oldtRelativePath.equalsIgnoreCase(name) && !oldtRelativePath.equalsIgnoreCase(name + Path.SEPARATOR);
            if (!tableInSpecifiedLoc) {
                srcFs = wh.getFs(srcPath);
                // get new location
                Database db = msdb.getDatabase(newDbName);
                Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
                destPath = new Path(databasePath, newTblName);
                destFs = wh.getFs(destPath);
                newt.getSd().setLocation(destPath.toString());
                // check that src and dest are on the same file system
                if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
                    throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported");
                }
                try {
                    if (destFs.exists(destPath)) {
                        throw new InvalidOperationException("New location for this table " + newDbName + "." + newTblName + " already exists : " + destPath);
                    }
                    // check that src exists and also checks permissions necessary, rename src to dest
                    if (srcFs.exists(srcPath) && wh.renameDir(srcPath, destPath, true)) {
                        dataWasMoved = true;
                    }
                } catch (IOException | MetaException e) {
                    LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
                    throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name + " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
                }
            }
            if (isPartitionedTable) {
                String oldTblLocPath = srcPath.toUri().getPath();
                String newTblLocPath = dataWasMoved ? destPath.toUri().getPath() : null;
                // also the location field in partition
                parts = msdb.getPartitions(dbname, name, -1);
                Map<Partition, ColumnStatistics> columnStatsNeedUpdated = new HashMap<>();
                for (Partition part : parts) {
                    String oldPartLoc = part.getSd().getLocation();
                    if (dataWasMoved && oldPartLoc.contains(oldTblLocPath)) {
                        URI oldUri = new Path(oldPartLoc).toUri();
                        String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
                        Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
                        part.getSd().setLocation(newPartLocPath.toString());
                    }
                    part.setDbName(newDbName);
                    part.setTableName(newTblName);
                    ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, part.getValues(), part.getSd().getCols(), oldt, part, null);
                    if (colStats != null) {
                        columnStatsNeedUpdated.put(part, colStats);
                    }
                }
                msdb.alterTable(dbname, name, newt);
                // alterPartition is only for changing the partition location in the table rename
                if (dataWasMoved) {
                    int partsToProcess = parts.size();
                    int partitionBatchSize = MetastoreConf.getIntVar(handler.getConf(), MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX);
                    int batchStart = 0;
                    while (partsToProcess > 0) {
                        int batchEnd = Math.min(batchStart + partitionBatchSize, parts.size());
                        List<Partition> partBatch = parts.subList(batchStart, batchEnd);
                        int partBatchSize = partBatch.size();
                        partsToProcess -= partBatchSize;
                        batchStart += partBatchSize;
                        List<List<String>> partValues = new ArrayList<>(partBatchSize);
                        for (Partition part : partBatch) {
                            partValues.add(part.getValues());
                        }
                        msdb.alterPartitions(newDbName, newTblName, partValues, partBatch);
                    }
                }
                for (Entry<Partition, ColumnStatistics> partColStats : columnStatsNeedUpdated.entrySet()) {
                    ColumnStatistics newPartColStats = partColStats.getValue();
                    newPartColStats.getStatsDesc().setDbName(newDbName);
                    newPartColStats.getStatsDesc().setTableName(newTblName);
                    msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues());
                }
            } else {
                alterTableUpdateTableColumnStats(msdb, oldt, newt);
            }
        } else {
            // operations other than table rename
            if (MetaStoreUtils.requireCalStats(null, null, newt, environmentContext) && !isPartitionedTable) {
                Database db = msdb.getDatabase(newDbName);
                // Update table stats. For partitioned table, we update stats in alterPartition()
                MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext, false);
            }
            if (isPartitionedTable) {
                // Currently only column related changes can be cascaded in alter table
                if (!MetaStoreUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) {
                    parts = msdb.getPartitions(dbname, name, -1);
                    for (Partition part : parts) {
                        Partition oldPart = new Partition(part);
                        List<FieldSchema> oldCols = part.getSd().getCols();
                        part.getSd().setCols(newt.getSd().getCols());
                        ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, part.getValues(), oldCols, oldt, part, null);
                        assert (colStats == null);
                        if (cascade) {
                            msdb.alterPartition(dbname, name, part.getValues(), part);
                        } else {
                            // update changed properties (stats)
                            oldPart.setParameters(part.getParameters());
                            msdb.alterPartition(dbname, name, part.getValues(), oldPart);
                        }
                    }
                    msdb.alterTable(dbname, name, newt);
                } else {
                    LOG.warn("Alter table not cascaded to partitions.");
                    alterTableUpdateTableColumnStats(msdb, oldt, newt);
                }
            } else {
                alterTableUpdateTableColumnStats(msdb, oldt, newt);
            }
        }
        if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
            if (oldt.getDbName().equalsIgnoreCase(newt.getDbName())) {
                MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_TABLE, new AlterTableEvent(oldt, newt, false, true, handler), environmentContext);
            } else {
                MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.DROP_TABLE, new DropTableEvent(oldt, true, false, handler), environmentContext);
                MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.CREATE_TABLE, new CreateTableEvent(newt, true, handler), environmentContext);
                if (isPartitionedTable) {
                    parts = msdb.getPartitions(newt.getDbName(), newt.getTableName(), -1);
                    MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ADD_PARTITION, new AddPartitionEvent(newt, parts, true, handler), environmentContext);
                }
            }
        }
        // commit the changes
        success = msdb.commitTransaction();
    } catch (InvalidObjectException e) {
        LOG.debug("Failed to get object from Metastore ", e);
        throw new InvalidOperationException("Unable to change partition or table." + " Check metastore logs for detailed stack." + e.getMessage());
    } catch (InvalidInputException e) {
        LOG.debug("Accessing Metastore failed due to invalid input ", e);
        throw new InvalidOperationException("Unable to change partition or table." + " Check metastore logs for detailed stack." + e.getMessage());
    } catch (NoSuchObjectException e) {
        LOG.debug("Object not found in metastore ", e);
        throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
    } finally {
        if (!success) {
            LOG.error("Failed to alter table " + dbname + "." + name);
            msdb.rollbackTransaction();
            if (dataWasMoved) {
                try {
                    if (destFs.exists(destPath)) {
                        if (!destFs.rename(destPath, srcPath)) {
                            LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
                        }
                    }
                } catch (IOException e) {
                    LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
                }
            }
        }
    }
}

Example 43 with InvalidObjectException

use of org.apache.hadoop.hive.metastore.api.InvalidObjectException in project hive by apache.

the class StatObjectConverter method convertToMTableColumnStatistics.

// JDO
public static MTableColumnStatistics convertToMTableColumnStatistics(MTable table, ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj) throws NoSuchObjectException, MetaException, InvalidObjectException {
    if (statsObj == null || statsDesc == null) {
        throw new InvalidObjectException("Invalid column stats object");
    }
    MTableColumnStatistics mColStats = new MTableColumnStatistics();
    mColStats.setTable(table);
    mColStats.setDbName(statsDesc.getDbName());
    mColStats.setTableName(statsDesc.getTableName());
    mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
    mColStats.setColName(statsObj.getColName());
    mColStats.setColType(statsObj.getColType());
    if (statsObj.getStatsData().isSetBooleanStats()) {
        BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
        mColStats.setBooleanStats(boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null);
    } else if (statsObj.getStatsData().isSetLongStats()) {
        LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
        mColStats.setLongStats(longStats.isSetNumNulls() ? longStats.getNumNulls() : null, longStats.isSetNumDVs() ? longStats.getNumDVs() : null, longStats.isSetBitVectors() ? longStats.getBitVectors() : null, longStats.isSetLowValue() ? longStats.getLowValue() : null, longStats.isSetHighValue() ? longStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDoubleStats()) {
        DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
        mColStats.setDoubleStats(doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null, doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDecimalStats()) {
        DecimalColumnStatsData decimalStats = statsObj.getStatsData().getDecimalStats();
        String low = decimalStats.isSetLowValue() ? createJdoDecimalString(decimalStats.getLowValue()) : null;
        String high = decimalStats.isSetHighValue() ? createJdoDecimalString(decimalStats.getHighValue()) : null;
        mColStats.setDecimalStats(decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null, decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null, low, high);
    } else if (statsObj.getStatsData().isSetStringStats()) {
        StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
        mColStats.setStringStats(stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null, stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, stringStats.isSetBitVectors() ? stringStats.getBitVectors() : null, stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetBinaryStats()) {
        BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
        mColStats.setBinaryStats(binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null, binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetDateStats()) {
        DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats();
        mColStats.setDateStats(dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null, dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null, dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null, dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null, dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null);
    }
    return mColStats;
}

Example 44 with InvalidObjectException

use of org.apache.hadoop.hive.metastore.api.InvalidObjectException in project hive by apache.

the class TestFunctions method testAlterFunctionInvalidNameInNew.

@Test
public void testAlterFunctionInvalidNameInNew() throws Exception {
    Function newFunction = getNewFunction();
    newFunction.setFunctionName("test_function_2;");
    try {
        client.alterFunction(DEFAULT_DATABASE, "test_function_to_find_2", newFunction);
        // TODO: Should have a check on the server side. Embedded metastore throws
        // InvalidObjectException, remote throws TApplicationException
        Assert.fail("Expected an InvalidObjectException or TApplicationException to be thrown");
    } catch (InvalidObjectException exception) {
    // Expected exception - Embedded MetaStore
    } catch (TApplicationException exception) {
    // Expected exception - Remote MetaStore
    }
}

Example 45 with InvalidObjectException

use of org.apache.hadoop.hive.metastore.api.InvalidObjectException in project hive by apache.

the class TestFunctions method testAlterFunctionEmptyNameInNew.

@Test
public void testAlterFunctionEmptyNameInNew() throws Exception {
    Function newFunction = getNewFunction();
    newFunction.setFunctionName("");
    try {
        client.alterFunction(DEFAULT_DATABASE, "test_function_to_find_2", newFunction);
        // TODO: Should have a check on the server side. Embedded metastore throws
        // InvalidObjectException, remote throws TApplicationException
        Assert.fail("Expected an InvalidObjectException or TApplicationException to be thrown");
    } catch (InvalidObjectException exception) {
    // Expected exception - Embedded MetaStore
    } catch (TApplicationException exception) {
    // Expected exception - Remote MetaStore
    }
}