Search in sources :

Example 1 with AlterTableEvent

use of org.apache.hadoop.hive.metastore.events.AlterTableEvent in project hive by apache.

the class HiveAlterHandler method alterTable.

@Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, MetaException {
    final boolean cascade = environmentContext != null && environmentContext.isSetProperties() && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(StatsSetupConst.CASCADE));
    if (newt == null) {
        throw new InvalidOperationException("New table is invalid: " + newt);
    }
    if (!MetaStoreUtils.validateName(newt.getTableName(), hiveConf)) {
        throw new InvalidOperationException(newt.getTableName() + " is not a valid object name");
    }
    String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols());
    if (validate != null) {
        throw new InvalidOperationException("Invalid column " + validate);
    }
    Path srcPath = null;
    FileSystem srcFs = null;
    Path destPath = null;
    FileSystem destFs = null;
    boolean success = false;
    boolean dataWasMoved = false;
    boolean rename = false;
    Table oldt = null;
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    try {
        msdb.openTransaction();
        name = name.toLowerCase();
        dbname = dbname.toLowerCase();
        // check if table with the new name already exists
        if (!newt.getTableName().equalsIgnoreCase(name) || !newt.getDbName().equalsIgnoreCase(dbname)) {
            if (msdb.getTable(newt.getDbName(), newt.getTableName()) != null) {
                throw new InvalidOperationException("new table " + newt.getDbName() + "." + newt.getTableName() + " already exists");
            }
            rename = true;
        }
        // get old table
        oldt = msdb.getTable(dbname, name);
        if (oldt == null) {
            throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
        }
        if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, false)) {
            // Throws InvalidOperationException if the new column types are not
            // compatible with the current column types.
            MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange(oldt.getSd().getCols(), newt.getSd().getCols());
        }
        if (cascade) {
            //Currently only column related changes can be cascaded in alter table
            if (MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
                List<Partition> parts = msdb.getPartitions(dbname, name, -1);
                for (Partition part : parts) {
                    List<FieldSchema> oldCols = part.getSd().getCols();
                    part.getSd().setCols(newt.getSd().getCols());
                    String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
                    updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part);
                    msdb.alterPartition(dbname, name, part.getValues(), part);
                }
            } else {
                LOG.warn("Alter table does not cascade changes to its partitions.");
            }
        }
        //check that partition keys have not changed, except for virtual views
        //however, allow the partition comments to change
        boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(), newt.getPartitionKeys());
        if (!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) {
            if (oldt.getPartitionKeys().size() != newt.getPartitionKeys().size() || !partKeysPartiallyEqual) {
                throw new InvalidOperationException("partition keys can not be changed.");
            }
        }
        // 4) the table was not initially created with a specified location
        if (rename && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 || StringUtils.isEmpty(newt.getSd().getLocation())) && !MetaStoreUtils.isExternalTable(oldt)) {
            Database olddb = msdb.getDatabase(dbname);
            // if a table was created in a user specified location using the DDL like
            // create table tbl ... location ...., it should be treated like an external table
            // in the table rename, its data location should not be changed. We can check
            // if the table directory was created directly under its database directory to tell
            // if it is such a table
            srcPath = new Path(oldt.getSd().getLocation());
            String oldtRelativePath = (new Path(olddb.getLocationUri()).toUri()).relativize(srcPath.toUri()).toString();
            boolean tableInSpecifiedLoc = !oldtRelativePath.equalsIgnoreCase(name) && !oldtRelativePath.equalsIgnoreCase(name + Path.SEPARATOR);
            if (!tableInSpecifiedLoc) {
                srcFs = wh.getFs(srcPath);
                // get new location
                Database db = msdb.getDatabase(newt.getDbName());
                Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
                destPath = new Path(databasePath, newt.getTableName().toLowerCase());
                destFs = wh.getFs(destPath);
                newt.getSd().setLocation(destPath.toString());
                // check that src and dest are on the same file system
                if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
                    throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported");
                }
                try {
                    if (destFs.exists(destPath)) {
                        throw new InvalidOperationException("New location for this table " + newt.getDbName() + "." + newt.getTableName() + " already exists : " + destPath);
                    }
                    // check that src exists and also checks permissions necessary, rename src to dest
                    if (srcFs.exists(srcPath) && srcFs.rename(srcPath, destPath)) {
                        dataWasMoved = true;
                    }
                } catch (IOException e) {
                    LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
                    throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name + " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
                }
                String oldTblLocPath = srcPath.toUri().getPath();
                String newTblLocPath = destPath.toUri().getPath();
                // also the location field in partition
                List<Partition> parts = msdb.getPartitions(dbname, name, -1);
                for (Partition part : parts) {
                    String oldPartLoc = part.getSd().getLocation();
                    if (oldPartLoc.contains(oldTblLocPath)) {
                        URI oldUri = new Path(oldPartLoc).toUri();
                        String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
                        Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
                        part.getSd().setLocation(newPartLocPath.toString());
                        String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
                        try {
                            //existing partition column stats is no longer valid, remove them
                            msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null);
                        } catch (InvalidInputException iie) {
                            throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
                        }
                        msdb.alterPartition(dbname, name, part.getValues(), part);
                    }
                }
            }
        } else if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext) && (newt.getPartitionKeysSize() == 0)) {
            Database db = msdb.getDatabase(newt.getDbName());
            // Update table stats. For partitioned table, we update stats in
            // alterPartition()
            MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext);
        }
        alterTableUpdateTableColumnStats(msdb, oldt, newt);
        if (transactionalListeners != null && transactionalListeners.size() > 0) {
            AlterTableEvent alterTableEvent = new AlterTableEvent(oldt, newt, true, handler);
            alterTableEvent.setEnvironmentContext(environmentContext);
            for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                transactionalListener.onAlterTable(alterTableEvent);
            }
        }
        // commit the changes
        success = msdb.commitTransaction();
    } catch (InvalidObjectException e) {
        LOG.debug("Failed to get object from Metastore ", e);
        throw new InvalidOperationException("Unable to change partition or table." + " Check metastore logs for detailed stack." + e.getMessage());
    } catch (NoSuchObjectException e) {
        LOG.debug("Object not found in metastore ", e);
        throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
    } finally {
        if (!success) {
            LOG.error("Failed to alter table " + dbname + "." + name);
            msdb.rollbackTransaction();
            if (dataWasMoved) {
                try {
                    if (destFs.exists(destPath)) {
                        if (!destFs.rename(destPath, srcPath)) {
                            LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
                        }
                    }
                } catch (IOException e) {
                    LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
                }
            }
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Partition(org.apache.hadoop.hive.metastore.api.Partition) InvalidInputException(org.apache.hadoop.hive.metastore.api.InvalidInputException) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) IOException(java.io.IOException) URI(java.net.URI) AlterTableEvent(org.apache.hadoop.hive.metastore.events.AlterTableEvent) FileSystem(org.apache.hadoop.fs.FileSystem) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) Database(org.apache.hadoop.hive.metastore.api.Database) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException)

Example 2 with AlterTableEvent

use of org.apache.hadoop.hive.metastore.events.AlterTableEvent in project hive by apache.

the class TestHiveMetaStoreWithEnvironmentContext method testEnvironmentContext.

public void testEnvironmentContext() throws Exception {
    int listSize = 0;
    List<ListenerEvent> notifyList = DummyListener.notifyList;
    assertEquals(notifyList.size(), listSize);
    msc.createDatabase(db);
    listSize++;
    assertEquals(listSize, notifyList.size());
    CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
    assert dbEvent.getStatus();
    msc.createTable(table, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
    assert tblEvent.getStatus();
    assertEquals(envContext, tblEvent.getEnvironmentContext());
    table = msc.getTable(dbName, tblName);
    partition.getSd().setLocation(table.getSd().getLocation() + "/part1");
    msc.add_partition(partition, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    assert partEvent.getStatus();
    assertEquals(envContext, partEvent.getEnvironmentContext());
    List<String> partVals = new ArrayList<String>();
    partVals.add("2012");
    msc.appendPartition(dbName, tblName, partVals, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    assert appendPartEvent.getStatus();
    assertEquals(envContext, appendPartEvent.getEnvironmentContext());
    table.setTableName(renamed);
    msc.alter_table_with_environmentContext(dbName, tblName, table, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    AlterTableEvent alterTableEvent = (AlterTableEvent) notifyList.get(listSize - 1);
    assert alterTableEvent.getStatus();
    assertEquals(envContext, alterTableEvent.getEnvironmentContext());
    table.setTableName(tblName);
    msc.alter_table_with_environmentContext(dbName, renamed, table, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    List<String> dropPartVals = new ArrayList<String>();
    dropPartVals.add("2011");
    msc.dropPartition(dbName, tblName, dropPartVals, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    DropPartitionEvent dropPartEvent = (DropPartitionEvent) notifyList.get(listSize - 1);
    assert dropPartEvent.getStatus();
    assertEquals(envContext, dropPartEvent.getEnvironmentContext());
    msc.dropPartition(dbName, tblName, "b=2012", true, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    DropPartitionEvent dropPartByNameEvent = (DropPartitionEvent) notifyList.get(listSize - 1);
    assert dropPartByNameEvent.getStatus();
    assertEquals(envContext, dropPartByNameEvent.getEnvironmentContext());
    msc.dropTable(dbName, tblName, true, false, envContext);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    DropTableEvent dropTblEvent = (DropTableEvent) notifyList.get(listSize - 1);
    assert dropTblEvent.getStatus();
    assertEquals(envContext, dropTblEvent.getEnvironmentContext());
    msc.dropDatabase(dbName);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
    assert dropDB.getStatus();
}
Also used : DropPartitionEvent(org.apache.hadoop.hive.metastore.events.DropPartitionEvent) CreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent) ArrayList(java.util.ArrayList) ListenerEvent(org.apache.hadoop.hive.metastore.events.ListenerEvent) AlterTableEvent(org.apache.hadoop.hive.metastore.events.AlterTableEvent) DropDatabaseEvent(org.apache.hadoop.hive.metastore.events.DropDatabaseEvent) CreateTableEvent(org.apache.hadoop.hive.metastore.events.CreateTableEvent) DropTableEvent(org.apache.hadoop.hive.metastore.events.DropTableEvent) AddPartitionEvent(org.apache.hadoop.hive.metastore.events.AddPartitionEvent)

Example 3 with AlterTableEvent

use of org.apache.hadoop.hive.metastore.events.AlterTableEvent in project hive by apache.

the class TestMetaStoreEventListener method testListener.

public void testListener() throws Exception {
    int listSize = 0;
    List<ListenerEvent> notifyList = DummyListener.notifyList;
    List<PreEventContext> preNotifyList = DummyPreListener.notifyList;
    assertEquals(notifyList.size(), listSize);
    assertEquals(preNotifyList.size(), listSize);
    driver.run("create database " + dbName);
    listSize++;
    PreCreateDatabaseEvent preDbEvent = (PreCreateDatabaseEvent) (preNotifyList.get(preNotifyList.size() - 1));
    Database db = msc.getDatabase(dbName);
    assertEquals(listSize, notifyList.size());
    assertEquals(listSize + 1, preNotifyList.size());
    validateCreateDb(db, preDbEvent.getDatabase());
    CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
    assert dbEvent.getStatus();
    validateCreateDb(db, dbEvent.getDatabase());
    driver.run("use " + dbName);
    driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
    PreCreateTableEvent preTblEvent = (PreCreateTableEvent) (preNotifyList.get(preNotifyList.size() - 1));
    listSize++;
    Table tbl = msc.getTable(dbName, tblName);
    validateCreateTable(tbl, preTblEvent.getTable());
    assertEquals(notifyList.size(), listSize);
    CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
    assert tblEvent.getStatus();
    validateCreateTable(tbl, tblEvent.getTable());
    driver.run("create index tmptbl_i on table tmptbl(a) as 'compact' " + "WITH DEFERRED REBUILD IDXPROPERTIES ('prop1'='val1', 'prop2'='val2')");
    // creates index table internally
    listSize += 2;
    assertEquals(notifyList.size(), listSize);
    AddIndexEvent addIndexEvent = (AddIndexEvent) notifyList.get(listSize - 1);
    assert addIndexEvent.getStatus();
    PreAddIndexEvent preAddIndexEvent = (PreAddIndexEvent) (preNotifyList.get(preNotifyList.size() - 3));
    Index oldIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
    validateAddIndex(oldIndex, addIndexEvent.getIndex());
    validateAddIndex(oldIndex, preAddIndexEvent.getIndex());
    driver.run("alter index tmptbl_i on tmptbl set IDXPROPERTIES " + "('prop1'='val1_new', 'prop3'='val3')");
    listSize++;
    assertEquals(notifyList.size(), listSize);
    Index newIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
    AlterIndexEvent alterIndexEvent = (AlterIndexEvent) notifyList.get(listSize - 1);
    assert alterIndexEvent.getStatus();
    validateAlterIndex(oldIndex, alterIndexEvent.getOldIndex(), newIndex, alterIndexEvent.getNewIndex());
    PreAlterIndexEvent preAlterIndexEvent = (PreAlterIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
    validateAlterIndex(oldIndex, preAlterIndexEvent.getOldIndex(), newIndex, preAlterIndexEvent.getNewIndex());
    driver.run("drop index tmptbl_i on tmptbl");
    listSize++;
    assertEquals(notifyList.size(), listSize);
    DropIndexEvent dropIndexEvent = (DropIndexEvent) notifyList.get(listSize - 1);
    assert dropIndexEvent.getStatus();
    validateDropIndex(newIndex, dropIndexEvent.getIndex());
    PreDropIndexEvent preDropIndexEvent = (PreDropIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
    validateDropIndex(newIndex, preDropIndexEvent.getIndex());
    driver.run("alter table tmptbl add partition (b='2011')");
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreAddPartitionEvent prePartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
    AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    assert partEvent.getStatus();
    Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011");
    Partition partAdded = partEvent.getPartitionIterator().next();
    validateAddPartition(part, partAdded);
    validateTableInAddPartition(tbl, partEvent.getTable());
    validateAddPartition(part, prePartEvent.getPartitions().get(0));
    // Test adding multiple partitions in a single partition-set, atomically.
    int currentTime = (int) System.currentTimeMillis();
    HiveMetaStoreClient hmsClient = new HiveMetaStoreClient(hiveConf);
    Table table = hmsClient.getTable(dbName, "tmptbl");
    Partition partition1 = new Partition(Arrays.asList("20110101"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
    Partition partition2 = new Partition(Arrays.asList("20110102"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
    Partition partition3 = new Partition(Arrays.asList("20110103"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
    hmsClient.add_partitions(Arrays.asList(partition1, partition2, partition3));
    ++listSize;
    AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    assertEquals("Unexpected table value.", table, multiplePartitionEvent.getTable());
    List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator());
    assertEquals("Unexpected number of partitions in event!", 3, multiParts.size());
    assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues());
    assertEquals("Unexpected partition value.", partition2.getValues(), multiParts.get(1).getValues());
    assertEquals("Unexpected partition value.", partition3.getValues(), multiParts.get(2).getValues());
    driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreAlterPartitionEvent preAlterPartEvent = (PreAlterPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
    //the partition did not change,
    // so the new partition should be similar to the original partition
    Partition origP = msc.getPartition(dbName, tblName, "b=2011");
    AlterPartitionEvent alterPartEvent = (AlterPartitionEvent) notifyList.get(listSize - 1);
    assert alterPartEvent.getStatus();
    validateAlterPartition(origP, origP, alterPartEvent.getOldPartition().getDbName(), alterPartEvent.getOldPartition().getTableName(), alterPartEvent.getOldPartition().getValues(), alterPartEvent.getNewPartition());
    validateAlterPartition(origP, origP, preAlterPartEvent.getDbName(), preAlterPartEvent.getTableName(), preAlterPartEvent.getNewPartition().getValues(), preAlterPartEvent.getNewPartition());
    List<String> part_vals = new ArrayList<String>();
    part_vals.add("c=2012");
    int preEventListSize;
    preEventListSize = preNotifyList.size() + 1;
    Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    assertEquals(preNotifyList.size(), preEventListSize);
    AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    Partition partAppended = appendPartEvent.getPartitionIterator().next();
    validateAddPartition(newPart, partAppended);
    PreAddPartitionEvent preAppendPartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
    validateAddPartition(newPart, preAppendPartEvent.getPartitions().get(0));
    driver.run(String.format("alter table %s rename to %s", tblName, renamed));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreAlterTableEvent preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
    Table renamedTable = msc.getTable(dbName, renamed);
    AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
    assert alterTableE.getStatus();
    validateAlterTable(tbl, renamedTable, alterTableE.getOldTable(), alterTableE.getNewTable());
    validateAlterTable(tbl, renamedTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
    //change the table name back
    driver.run(String.format("alter table %s rename to %s", renamed, tblName));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    driver.run(String.format("alter table %s ADD COLUMNS (c int)", tblName));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
    Table altTable = msc.getTable(dbName, tblName);
    alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
    assert alterTableE.getStatus();
    validateAlterTableColumns(tbl, altTable, alterTableE.getOldTable(), alterTableE.getNewTable());
    validateAlterTableColumns(tbl, altTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
    Map<String, String> kvs = new HashMap<String, String>(1);
    kvs.put("b", "2011");
    msc.markPartitionForEvent("hive2038", "tmptbl", kvs, PartitionEventType.LOAD_DONE);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent) notifyList.get(listSize - 1);
    assert partMarkEvent.getStatus();
    validateLoadPartitionDone("tmptbl", kvs, partMarkEvent.getTable().getTableName(), partMarkEvent.getPartitionName());
    PreLoadPartitionDoneEvent prePartMarkEvent = (PreLoadPartitionDoneEvent) preNotifyList.get(preNotifyList.size() - 1);
    validateLoadPartitionDone("tmptbl", kvs, prePartMarkEvent.getTableName(), prePartMarkEvent.getPartitionName());
    driver.run(String.format("alter table %s drop partition (b='2011')", tblName));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreDropPartitionEvent preDropPart = (PreDropPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
    DropPartitionEvent dropPart = (DropPartitionEvent) notifyList.get(listSize - 1);
    assert dropPart.getStatus();
    validateDropPartition(Collections.singletonList(part).iterator(), dropPart.getPartitionIterator());
    validateTableInDropPartition(tbl, dropPart.getTable());
    validateDropPartition(Collections.singletonList(part).iterator(), preDropPart.getPartitionIterator());
    validateTableInDropPartition(tbl, preDropPart.getTable());
    driver.run("drop table " + tblName);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreDropTableEvent preDropTbl = (PreDropTableEvent) preNotifyList.get(preNotifyList.size() - 1);
    DropTableEvent dropTbl = (DropTableEvent) notifyList.get(listSize - 1);
    assert dropTbl.getStatus();
    validateDropTable(tbl, dropTbl.getTable());
    validateDropTable(tbl, preDropTbl.getTable());
    driver.run("drop database " + dbName);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreDropDatabaseEvent preDropDB = (PreDropDatabaseEvent) preNotifyList.get(preNotifyList.size() - 1);
    DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
    assert dropDB.getStatus();
    validateDropDb(db, dropDB.getDatabase());
    validateDropDb(db, preDropDB.getDatabase());
    SetProcessor.setVariable("metaconf:hive.metastore.try.direct.sql", "false");
    ConfigChangeEvent event = (ConfigChangeEvent) notifyList.get(notifyList.size() - 1);
    assertEquals("hive.metastore.try.direct.sql", event.getKey());
    assertEquals("true", event.getOldValue());
    assertEquals("false", event.getNewValue());
}
Also used : PreAddPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent) PreDropIndexEvent(org.apache.hadoop.hive.metastore.events.PreDropIndexEvent) HashMap(java.util.HashMap) CreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent) PreCreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent) PreAlterPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent) ArrayList(java.util.ArrayList) Index(org.apache.hadoop.hive.metastore.api.Index) PreAlterTableEvent(org.apache.hadoop.hive.metastore.events.PreAlterTableEvent) PreLoadPartitionDoneEvent(org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent) PreDropDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent) ListenerEvent(org.apache.hadoop.hive.metastore.events.ListenerEvent) AlterTableEvent(org.apache.hadoop.hive.metastore.events.AlterTableEvent) PreAlterTableEvent(org.apache.hadoop.hive.metastore.events.PreAlterTableEvent) DropDatabaseEvent(org.apache.hadoop.hive.metastore.events.DropDatabaseEvent) PreDropDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent) PreAddIndexEvent(org.apache.hadoop.hive.metastore.events.PreAddIndexEvent) PreCreateTableEvent(org.apache.hadoop.hive.metastore.events.PreCreateTableEvent) CreateTableEvent(org.apache.hadoop.hive.metastore.events.CreateTableEvent) PreAlterIndexEvent(org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent) DropIndexEvent(org.apache.hadoop.hive.metastore.events.DropIndexEvent) PreDropIndexEvent(org.apache.hadoop.hive.metastore.events.PreDropIndexEvent) PreLoadPartitionDoneEvent(org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent) LoadPartitionDoneEvent(org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent) PreDropTableEvent(org.apache.hadoop.hive.metastore.events.PreDropTableEvent) DropTableEvent(org.apache.hadoop.hive.metastore.events.DropTableEvent) Database(org.apache.hadoop.hive.metastore.api.Database) PreCreateTableEvent(org.apache.hadoop.hive.metastore.events.PreCreateTableEvent) Partition(org.apache.hadoop.hive.metastore.api.Partition) PreAddIndexEvent(org.apache.hadoop.hive.metastore.events.PreAddIndexEvent) AddIndexEvent(org.apache.hadoop.hive.metastore.events.AddIndexEvent) PreDropPartitionEvent(org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent) DropPartitionEvent(org.apache.hadoop.hive.metastore.events.DropPartitionEvent) Table(org.apache.hadoop.hive.metastore.api.Table) AlterPartitionEvent(org.apache.hadoop.hive.metastore.events.AlterPartitionEvent) PreAlterPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent) PreCreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent) PreDropPartitionEvent(org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent) PreEventContext(org.apache.hadoop.hive.metastore.events.PreEventContext) PreDropTableEvent(org.apache.hadoop.hive.metastore.events.PreDropTableEvent) ConfigChangeEvent(org.apache.hadoop.hive.metastore.events.ConfigChangeEvent) PreAddPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent) AddPartitionEvent(org.apache.hadoop.hive.metastore.events.AddPartitionEvent) PreAlterIndexEvent(org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent) AlterIndexEvent(org.apache.hadoop.hive.metastore.events.AlterIndexEvent)

Aggregations

AlterTableEvent (org.apache.hadoop.hive.metastore.events.AlterTableEvent)3 ArrayList (java.util.ArrayList)2 Database (org.apache.hadoop.hive.metastore.api.Database)2 Partition (org.apache.hadoop.hive.metastore.api.Partition)2 Table (org.apache.hadoop.hive.metastore.api.Table)2 AddPartitionEvent (org.apache.hadoop.hive.metastore.events.AddPartitionEvent)2 CreateDatabaseEvent (org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent)2 CreateTableEvent (org.apache.hadoop.hive.metastore.events.CreateTableEvent)2 DropDatabaseEvent (org.apache.hadoop.hive.metastore.events.DropDatabaseEvent)2 DropPartitionEvent (org.apache.hadoop.hive.metastore.events.DropPartitionEvent)2 DropTableEvent (org.apache.hadoop.hive.metastore.events.DropTableEvent)2 ListenerEvent (org.apache.hadoop.hive.metastore.events.ListenerEvent)2 IOException (java.io.IOException)1 URI (java.net.URI)1 HashMap (java.util.HashMap)1 FileSystem (org.apache.hadoop.fs.FileSystem)1 Path (org.apache.hadoop.fs.Path)1 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)1 Index (org.apache.hadoop.hive.metastore.api.Index)1 InvalidInputException (org.apache.hadoop.hive.metastore.api.InvalidInputException)1