Search in sources :

Example 1 with AlterPartitionEvent

use of org.apache.hadoop.hive.metastore.events.AlterPartitionEvent in project hive by apache.

the class HiveAlterHandler method alterPartitions.

@Override
public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<Partition> new_parts, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
    List<Partition> oldParts = new ArrayList<Partition>();
    List<List<String>> partValsList = new ArrayList<List<String>>();
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    Table tbl = msdb.getTable(dbname, name);
    if (tbl == null) {
        throw new InvalidObjectException("Unable to alter partitions because table or database does not exist.");
    }
    boolean success = false;
    try {
        msdb.openTransaction();
        for (Partition tmpPart : new_parts) {
            // Set DDL time to now if not specified
            if (tmpPart.getParameters() == null || tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
                tmpPart.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
            }
            Partition oldTmpPart = msdb.getPartition(dbname, name, tmpPart.getValues());
            oldParts.add(oldTmpPart);
            partValsList.add(tmpPart.getValues());
            if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) {
                // Check if stats are same, no need to update
                if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) {
                    MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters());
                } else {
                    MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
                }
            }
            updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
        }
        msdb.alterPartitions(dbname, name, partValsList, new_parts);
        Iterator<Partition> oldPartsIt = oldParts.iterator();
        for (Partition newPart : new_parts) {
            Partition oldPart;
            if (oldPartsIt.hasNext()) {
                oldPart = oldPartsIt.next();
            } else {
                throw new InvalidOperationException("Missing old partition corresponding to new partition " + "when invoking MetaStoreEventListener for alterPartitions event.");
            }
            if (transactionalListeners != null && transactionalListeners.size() > 0) {
                AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, newPart, tbl, true, handler);
                for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                    transactionalListener.onAlterPartition(alterPartitionEvent);
                }
            }
        }
        success = msdb.commitTransaction();
    } catch (InvalidObjectException | NoSuchObjectException e) {
        throw new InvalidOperationException("Alter partition operation failed: " + e);
    } finally {
        if (!success) {
            msdb.rollbackTransaction();
        }
    }
    return oldParts;
}
Also used : Partition(org.apache.hadoop.hive.metastore.api.Partition) Table(org.apache.hadoop.hive.metastore.api.Table) AlterPartitionEvent(org.apache.hadoop.hive.metastore.events.AlterPartitionEvent) ArrayList(java.util.ArrayList) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) ArrayList(java.util.ArrayList) List(java.util.List) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException)

Example 2 with AlterPartitionEvent

use of org.apache.hadoop.hive.metastore.events.AlterPartitionEvent in project hive by apache.

the class HiveAlterHandler method alterPartition.

@Override
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<String> part_vals, final Partition new_part, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
    boolean success = false;
    Path srcPath = null;
    Path destPath = null;
    FileSystem srcFs = null;
    FileSystem destFs;
    Partition oldPart = null;
    String oldPartLoc = null;
    String newPartLoc = null;
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    // Set DDL time to now if not specified
    if (new_part.getParameters() == null || new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
        new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
    }
    Table tbl = msdb.getTable(dbname, name);
    if (tbl == null) {
        throw new InvalidObjectException("Unable to alter partition because table or database does not exist.");
    }
    //alter partition
    if (part_vals == null || part_vals.size() == 0) {
        try {
            msdb.openTransaction();
            oldPart = msdb.getPartition(dbname, name, new_part.getValues());
            if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
                // if stats are same, no need to update
                if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) {
                    MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters());
                } else {
                    MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
                }
            }
            updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
            msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
            if (transactionalListeners != null && transactionalListeners.size() > 0) {
                AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
                alterPartitionEvent.setEnvironmentContext(environmentContext);
                for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                    transactionalListener.onAlterPartition(alterPartitionEvent);
                }
            }
            success = msdb.commitTransaction();
        } catch (InvalidObjectException e) {
            throw new InvalidOperationException("alter is not possible");
        } catch (NoSuchObjectException e) {
            //old partition does not exist
            throw new InvalidOperationException("alter is not possible");
        } finally {
            if (!success) {
                msdb.rollbackTransaction();
            }
        }
        return oldPart;
    }
    //rename partition
    try {
        msdb.openTransaction();
        try {
            oldPart = msdb.getPartition(dbname, name, part_vals);
        } catch (NoSuchObjectException e) {
            // this means there is no existing partition
            throw new InvalidObjectException("Unable to rename partition because old partition does not exist");
        }
        Partition check_part;
        try {
            check_part = msdb.getPartition(dbname, name, new_part.getValues());
        } catch (NoSuchObjectException e) {
            // this means there is no existing partition
            check_part = null;
        }
        if (check_part != null) {
            throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." + new_part.getValues());
        }
        // if the external partition is renamed, the file should not change
        if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
            new_part.getSd().setLocation(oldPart.getSd().getLocation());
            String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
            try {
                //existing partition column stats is no longer valid, remove
                msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
            } catch (NoSuchObjectException nsoe) {
            //ignore
            } catch (InvalidInputException iie) {
                throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
            }
            msdb.alterPartition(dbname, name, part_vals, new_part);
        } else {
            try {
                destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
                destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
            } catch (NoSuchObjectException e) {
                LOG.debug("Didn't find object in metastore ", e);
                throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
            }
            if (destPath != null) {
                newPartLoc = destPath.toString();
                oldPartLoc = oldPart.getSd().getLocation();
                LOG.info("srcPath:" + oldPartLoc);
                LOG.info("descPath:" + newPartLoc);
                srcPath = new Path(oldPartLoc);
                srcFs = wh.getFs(srcPath);
                destFs = wh.getFs(destPath);
                // check that src and dest are on the same file system
                if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
                    throw new InvalidOperationException("New table location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported.");
                }
                try {
                    srcFs.exists(srcPath);
                    if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
                        throw new InvalidOperationException("New location for this table " + tbl.getDbName() + "." + tbl.getTableName() + " already exists : " + destPath);
                    }
                } catch (IOException e) {
                    throw new InvalidOperationException("Unable to access new location " + destPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues());
                }
                new_part.getSd().setLocation(newPartLoc);
                if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
                    MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
                }
                String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
                try {
                    //existing partition column stats is no longer valid, remove
                    msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
                } catch (NoSuchObjectException nsoe) {
                //ignore
                } catch (InvalidInputException iie) {
                    throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
                }
                msdb.alterPartition(dbname, name, part_vals, new_part);
            }
        }
        if (transactionalListeners != null && transactionalListeners.size() > 0) {
            AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
            alterPartitionEvent.setEnvironmentContext(environmentContext);
            for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                transactionalListener.onAlterPartition(alterPartitionEvent);
            }
        }
        success = msdb.commitTransaction();
    } finally {
        if (!success) {
            msdb.rollbackTransaction();
        }
        if (success && newPartLoc != null && newPartLoc.compareTo(oldPartLoc) != 0) {
            //rename the data directory
            try {
                if (srcFs.exists(srcPath)) {
                    //if destPath's parent path doesn't exist, we should mkdir it
                    Path destParentPath = destPath.getParent();
                    if (!wh.mkdirs(destParentPath, true)) {
                        throw new IOException("Unable to create path " + destParentPath);
                    }
                    wh.renameDir(srcPath, destPath, true);
                    LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
                }
            } catch (IOException ex) {
                LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, ex);
                boolean revertMetaDataTransaction = false;
                try {
                    msdb.openTransaction();
                    msdb.alterPartition(dbname, name, new_part.getValues(), oldPart);
                    if (transactionalListeners != null && transactionalListeners.size() > 0) {
                        AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(new_part, oldPart, tbl, true, handler);
                        alterPartitionEvent.setEnvironmentContext(environmentContext);
                        for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                            transactionalListener.onAlterPartition(alterPartitionEvent);
                        }
                    }
                    revertMetaDataTransaction = msdb.commitTransaction();
                } catch (Exception ex2) {
                    LOG.error("Attempt to revert partition metadata change failed. The revert was attempted " + "because associated filesystem rename operation failed with exception " + ex.getMessage(), ex2);
                    if (!revertMetaDataTransaction) {
                        msdb.rollbackTransaction();
                    }
                }
                throw new InvalidOperationException("Unable to access old location " + srcPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + part_vals);
            }
        }
    }
    return oldPart;
}
Also used : Path(org.apache.hadoop.fs.Path) Partition(org.apache.hadoop.hive.metastore.api.Partition) InvalidInputException(org.apache.hadoop.hive.metastore.api.InvalidInputException) Table(org.apache.hadoop.hive.metastore.api.Table) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) AlterPartitionEvent(org.apache.hadoop.hive.metastore.events.AlterPartitionEvent) IOException(java.io.IOException) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) InvalidInputException(org.apache.hadoop.hive.metastore.api.InvalidInputException) IOException(java.io.IOException) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) RemoteException(org.apache.hadoop.ipc.RemoteException) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) FileSystem(org.apache.hadoop.fs.FileSystem) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException)

Example 3 with AlterPartitionEvent

use of org.apache.hadoop.hive.metastore.events.AlterPartitionEvent in project hive by apache.

the class TestMetaStoreEventListener method testListener.

public void testListener() throws Exception {
    int listSize = 0;
    List<ListenerEvent> notifyList = DummyListener.notifyList;
    List<PreEventContext> preNotifyList = DummyPreListener.notifyList;
    assertEquals(notifyList.size(), listSize);
    assertEquals(preNotifyList.size(), listSize);
    driver.run("create database " + dbName);
    listSize++;
    PreCreateDatabaseEvent preDbEvent = (PreCreateDatabaseEvent) (preNotifyList.get(preNotifyList.size() - 1));
    Database db = msc.getDatabase(dbName);
    assertEquals(listSize, notifyList.size());
    assertEquals(listSize + 1, preNotifyList.size());
    validateCreateDb(db, preDbEvent.getDatabase());
    CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
    assert dbEvent.getStatus();
    validateCreateDb(db, dbEvent.getDatabase());
    driver.run("use " + dbName);
    driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
    PreCreateTableEvent preTblEvent = (PreCreateTableEvent) (preNotifyList.get(preNotifyList.size() - 1));
    listSize++;
    Table tbl = msc.getTable(dbName, tblName);
    validateCreateTable(tbl, preTblEvent.getTable());
    assertEquals(notifyList.size(), listSize);
    CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
    assert tblEvent.getStatus();
    validateCreateTable(tbl, tblEvent.getTable());
    driver.run("create index tmptbl_i on table tmptbl(a) as 'compact' " + "WITH DEFERRED REBUILD IDXPROPERTIES ('prop1'='val1', 'prop2'='val2')");
    // creates index table internally
    listSize += 2;
    assertEquals(notifyList.size(), listSize);
    AddIndexEvent addIndexEvent = (AddIndexEvent) notifyList.get(listSize - 1);
    assert addIndexEvent.getStatus();
    PreAddIndexEvent preAddIndexEvent = (PreAddIndexEvent) (preNotifyList.get(preNotifyList.size() - 3));
    Index oldIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
    validateAddIndex(oldIndex, addIndexEvent.getIndex());
    validateAddIndex(oldIndex, preAddIndexEvent.getIndex());
    driver.run("alter index tmptbl_i on tmptbl set IDXPROPERTIES " + "('prop1'='val1_new', 'prop3'='val3')");
    listSize++;
    assertEquals(notifyList.size(), listSize);
    Index newIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
    AlterIndexEvent alterIndexEvent = (AlterIndexEvent) notifyList.get(listSize - 1);
    assert alterIndexEvent.getStatus();
    validateAlterIndex(oldIndex, alterIndexEvent.getOldIndex(), newIndex, alterIndexEvent.getNewIndex());
    PreAlterIndexEvent preAlterIndexEvent = (PreAlterIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
    validateAlterIndex(oldIndex, preAlterIndexEvent.getOldIndex(), newIndex, preAlterIndexEvent.getNewIndex());
    driver.run("drop index tmptbl_i on tmptbl");
    listSize++;
    assertEquals(notifyList.size(), listSize);
    DropIndexEvent dropIndexEvent = (DropIndexEvent) notifyList.get(listSize - 1);
    assert dropIndexEvent.getStatus();
    validateDropIndex(newIndex, dropIndexEvent.getIndex());
    PreDropIndexEvent preDropIndexEvent = (PreDropIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
    validateDropIndex(newIndex, preDropIndexEvent.getIndex());
    driver.run("alter table tmptbl add partition (b='2011')");
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreAddPartitionEvent prePartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
    AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    assert partEvent.getStatus();
    Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011");
    Partition partAdded = partEvent.getPartitionIterator().next();
    validateAddPartition(part, partAdded);
    validateTableInAddPartition(tbl, partEvent.getTable());
    validateAddPartition(part, prePartEvent.getPartitions().get(0));
    // Test adding multiple partitions in a single partition-set, atomically.
    int currentTime = (int) System.currentTimeMillis();
    HiveMetaStoreClient hmsClient = new HiveMetaStoreClient(hiveConf);
    Table table = hmsClient.getTable(dbName, "tmptbl");
    Partition partition1 = new Partition(Arrays.asList("20110101"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
    Partition partition2 = new Partition(Arrays.asList("20110102"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
    Partition partition3 = new Partition(Arrays.asList("20110103"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
    hmsClient.add_partitions(Arrays.asList(partition1, partition2, partition3));
    ++listSize;
    AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    assertEquals("Unexpected table value.", table, multiplePartitionEvent.getTable());
    List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator());
    assertEquals("Unexpected number of partitions in event!", 3, multiParts.size());
    assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues());
    assertEquals("Unexpected partition value.", partition2.getValues(), multiParts.get(1).getValues());
    assertEquals("Unexpected partition value.", partition3.getValues(), multiParts.get(2).getValues());
    driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreAlterPartitionEvent preAlterPartEvent = (PreAlterPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
    //the partition did not change,
    // so the new partition should be similar to the original partition
    Partition origP = msc.getPartition(dbName, tblName, "b=2011");
    AlterPartitionEvent alterPartEvent = (AlterPartitionEvent) notifyList.get(listSize - 1);
    assert alterPartEvent.getStatus();
    validateAlterPartition(origP, origP, alterPartEvent.getOldPartition().getDbName(), alterPartEvent.getOldPartition().getTableName(), alterPartEvent.getOldPartition().getValues(), alterPartEvent.getNewPartition());
    validateAlterPartition(origP, origP, preAlterPartEvent.getDbName(), preAlterPartEvent.getTableName(), preAlterPartEvent.getNewPartition().getValues(), preAlterPartEvent.getNewPartition());
    List<String> part_vals = new ArrayList<String>();
    part_vals.add("c=2012");
    int preEventListSize;
    preEventListSize = preNotifyList.size() + 1;
    Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    assertEquals(preNotifyList.size(), preEventListSize);
    AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
    Partition partAppended = appendPartEvent.getPartitionIterator().next();
    validateAddPartition(newPart, partAppended);
    PreAddPartitionEvent preAppendPartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
    validateAddPartition(newPart, preAppendPartEvent.getPartitions().get(0));
    driver.run(String.format("alter table %s rename to %s", tblName, renamed));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreAlterTableEvent preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
    Table renamedTable = msc.getTable(dbName, renamed);
    AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
    assert alterTableE.getStatus();
    validateAlterTable(tbl, renamedTable, alterTableE.getOldTable(), alterTableE.getNewTable());
    validateAlterTable(tbl, renamedTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
    //change the table name back
    driver.run(String.format("alter table %s rename to %s", renamed, tblName));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    driver.run(String.format("alter table %s ADD COLUMNS (c int)", tblName));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
    Table altTable = msc.getTable(dbName, tblName);
    alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
    assert alterTableE.getStatus();
    validateAlterTableColumns(tbl, altTable, alterTableE.getOldTable(), alterTableE.getNewTable());
    validateAlterTableColumns(tbl, altTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
    Map<String, String> kvs = new HashMap<String, String>(1);
    kvs.put("b", "2011");
    msc.markPartitionForEvent("hive2038", "tmptbl", kvs, PartitionEventType.LOAD_DONE);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent) notifyList.get(listSize - 1);
    assert partMarkEvent.getStatus();
    validateLoadPartitionDone("tmptbl", kvs, partMarkEvent.getTable().getTableName(), partMarkEvent.getPartitionName());
    PreLoadPartitionDoneEvent prePartMarkEvent = (PreLoadPartitionDoneEvent) preNotifyList.get(preNotifyList.size() - 1);
    validateLoadPartitionDone("tmptbl", kvs, prePartMarkEvent.getTableName(), prePartMarkEvent.getPartitionName());
    driver.run(String.format("alter table %s drop partition (b='2011')", tblName));
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreDropPartitionEvent preDropPart = (PreDropPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
    DropPartitionEvent dropPart = (DropPartitionEvent) notifyList.get(listSize - 1);
    assert dropPart.getStatus();
    validateDropPartition(Collections.singletonList(part).iterator(), dropPart.getPartitionIterator());
    validateTableInDropPartition(tbl, dropPart.getTable());
    validateDropPartition(Collections.singletonList(part).iterator(), preDropPart.getPartitionIterator());
    validateTableInDropPartition(tbl, preDropPart.getTable());
    driver.run("drop table " + tblName);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreDropTableEvent preDropTbl = (PreDropTableEvent) preNotifyList.get(preNotifyList.size() - 1);
    DropTableEvent dropTbl = (DropTableEvent) notifyList.get(listSize - 1);
    assert dropTbl.getStatus();
    validateDropTable(tbl, dropTbl.getTable());
    validateDropTable(tbl, preDropTbl.getTable());
    driver.run("drop database " + dbName);
    listSize++;
    assertEquals(notifyList.size(), listSize);
    PreDropDatabaseEvent preDropDB = (PreDropDatabaseEvent) preNotifyList.get(preNotifyList.size() - 1);
    DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
    assert dropDB.getStatus();
    validateDropDb(db, dropDB.getDatabase());
    validateDropDb(db, preDropDB.getDatabase());
    SetProcessor.setVariable("metaconf:hive.metastore.try.direct.sql", "false");
    ConfigChangeEvent event = (ConfigChangeEvent) notifyList.get(notifyList.size() - 1);
    assertEquals("hive.metastore.try.direct.sql", event.getKey());
    assertEquals("true", event.getOldValue());
    assertEquals("false", event.getNewValue());
}
Also used : PreAddPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent) PreDropIndexEvent(org.apache.hadoop.hive.metastore.events.PreDropIndexEvent) HashMap(java.util.HashMap) CreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent) PreCreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent) PreAlterPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent) ArrayList(java.util.ArrayList) Index(org.apache.hadoop.hive.metastore.api.Index) PreAlterTableEvent(org.apache.hadoop.hive.metastore.events.PreAlterTableEvent) PreLoadPartitionDoneEvent(org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent) PreDropDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent) ListenerEvent(org.apache.hadoop.hive.metastore.events.ListenerEvent) AlterTableEvent(org.apache.hadoop.hive.metastore.events.AlterTableEvent) PreAlterTableEvent(org.apache.hadoop.hive.metastore.events.PreAlterTableEvent) DropDatabaseEvent(org.apache.hadoop.hive.metastore.events.DropDatabaseEvent) PreDropDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent) PreAddIndexEvent(org.apache.hadoop.hive.metastore.events.PreAddIndexEvent) PreCreateTableEvent(org.apache.hadoop.hive.metastore.events.PreCreateTableEvent) CreateTableEvent(org.apache.hadoop.hive.metastore.events.CreateTableEvent) PreAlterIndexEvent(org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent) DropIndexEvent(org.apache.hadoop.hive.metastore.events.DropIndexEvent) PreDropIndexEvent(org.apache.hadoop.hive.metastore.events.PreDropIndexEvent) PreLoadPartitionDoneEvent(org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent) LoadPartitionDoneEvent(org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent) PreDropTableEvent(org.apache.hadoop.hive.metastore.events.PreDropTableEvent) DropTableEvent(org.apache.hadoop.hive.metastore.events.DropTableEvent) Database(org.apache.hadoop.hive.metastore.api.Database) PreCreateTableEvent(org.apache.hadoop.hive.metastore.events.PreCreateTableEvent) Partition(org.apache.hadoop.hive.metastore.api.Partition) PreAddIndexEvent(org.apache.hadoop.hive.metastore.events.PreAddIndexEvent) AddIndexEvent(org.apache.hadoop.hive.metastore.events.AddIndexEvent) PreDropPartitionEvent(org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent) DropPartitionEvent(org.apache.hadoop.hive.metastore.events.DropPartitionEvent) Table(org.apache.hadoop.hive.metastore.api.Table) AlterPartitionEvent(org.apache.hadoop.hive.metastore.events.AlterPartitionEvent) PreAlterPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent) PreCreateDatabaseEvent(org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent) PreDropPartitionEvent(org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent) PreEventContext(org.apache.hadoop.hive.metastore.events.PreEventContext) PreDropTableEvent(org.apache.hadoop.hive.metastore.events.PreDropTableEvent) ConfigChangeEvent(org.apache.hadoop.hive.metastore.events.ConfigChangeEvent) PreAddPartitionEvent(org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent) AddPartitionEvent(org.apache.hadoop.hive.metastore.events.AddPartitionEvent) PreAlterIndexEvent(org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent) AlterIndexEvent(org.apache.hadoop.hive.metastore.events.AlterIndexEvent)

Aggregations

Partition (org.apache.hadoop.hive.metastore.api.Partition)3 Table (org.apache.hadoop.hive.metastore.api.Table)3 AlterPartitionEvent (org.apache.hadoop.hive.metastore.events.AlterPartitionEvent)3 ArrayList (java.util.ArrayList)2 InvalidObjectException (org.apache.hadoop.hive.metastore.api.InvalidObjectException)2 InvalidOperationException (org.apache.hadoop.hive.metastore.api.InvalidOperationException)2 NoSuchObjectException (org.apache.hadoop.hive.metastore.api.NoSuchObjectException)2 IOException (java.io.IOException)1 HashMap (java.util.HashMap)1 List (java.util.List)1 FileSystem (org.apache.hadoop.fs.FileSystem)1 Path (org.apache.hadoop.fs.Path)1 AlreadyExistsException (org.apache.hadoop.hive.metastore.api.AlreadyExistsException)1 Database (org.apache.hadoop.hive.metastore.api.Database)1 Index (org.apache.hadoop.hive.metastore.api.Index)1 InvalidInputException (org.apache.hadoop.hive.metastore.api.InvalidInputException)1 MetaException (org.apache.hadoop.hive.metastore.api.MetaException)1 AddIndexEvent (org.apache.hadoop.hive.metastore.events.AddIndexEvent)1 AddPartitionEvent (org.apache.hadoop.hive.metastore.events.AddPartitionEvent)1 AlterIndexEvent (org.apache.hadoop.hive.metastore.events.AlterIndexEvent)1