use of org.apache.hadoop.hive.metastore.messaging.AlterTableMessage in project hive by apache.
the class DbNotificationListener method onAlterTable.
/**
* @param tableEvent alter table event
* @throws MetaException
*/
@Override
public void onAlterTable(AlterTableEvent tableEvent) throws MetaException {
Table before = tableEvent.getOldTable();
Table after = tableEvent.getNewTable();
AlterTableMessage msg = MessageBuilder.getInstance().buildAlterTableMessage(before, after, tableEvent.getIsTruncateOp(), tableEvent.getWriteId());
NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_TABLE.toString(), msgEncoder.getSerializer().serialize(msg));
event.setCatName(after.isSetCatName() ? after.getCatName() : DEFAULT_CATALOG_NAME);
event.setDbName(after.getDbName());
event.setTableName(after.getTableName());
process(event, tableEvent);
}
use of org.apache.hadoop.hive.metastore.messaging.AlterTableMessage in project hive by apache.
the class TableHandler method extract.
private Tuple extract(Context context) throws SemanticException {
try {
String tableType = null;
long writeId = DEFAULT_WRITE_ID;
switch(context.dmd.getDumpType()) {
case EVENT_CREATE_TABLE:
case EVENT_ADD_PARTITION:
Path metadataPath = new Path(context.location, EximUtil.METADATA_NAME);
MetaData rv = EximUtil.readMetaData(metadataPath.getFileSystem(context.hiveConf), metadataPath);
tableType = rv.getTable().getTableType();
break;
case EVENT_ALTER_TABLE:
AlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(context.dmd.getPayload());
tableType = alterTableMessage.getTableObjAfter().getTableType();
writeId = alterTableMessage.getWriteId();
break;
case EVENT_ALTER_PARTITION:
AlterPartitionMessage msg = deserializer.getAlterPartitionMessage(context.dmd.getPayload());
tableType = msg.getTableObj().getTableType();
writeId = msg.getWriteId();
break;
default:
break;
}
boolean isExternalTable = tableType != null && TableType.EXTERNAL_TABLE.equals(Enum.valueOf(TableType.class, tableType));
return new Tuple(isExternalTable, writeId);
} catch (Exception e) {
LOG.error("failed to determine if the table associated with the event is external or not", e);
throw new SemanticException(e);
}
}
use of org.apache.hadoop.hive.metastore.messaging.AlterTableMessage in project hive by apache.
the class ReplicationSemanticAnalyzer method dumpEvent.
private void dumpEvent(NotificationEvent ev, Path evRoot, Path cmRoot) throws Exception {
long evid = ev.getEventId();
String evidStr = String.valueOf(evid);
ReplicationSpec replicationSpec = getNewEventOnlyReplicationSpec(evidStr);
MessageDeserializer md = MessageFactory.getInstance().getDeserializer();
switch(ev.getEventType()) {
case MessageFactory.CREATE_TABLE_EVENT:
{
CreateTableMessage ctm = md.getCreateTableMessage(ev.getMessage());
LOG.info("Processing#{} CREATE_TABLE message : {}", ev.getEventId(), ev.getMessage());
org.apache.hadoop.hive.metastore.api.Table tobj = ctm.getTableObj();
if (tobj == null) {
LOG.debug("Event#{} was a CREATE_TABLE_EVENT with no table listed");
break;
}
Table qlMdTable = new Table(tobj);
if (qlMdTable.isView()) {
replicationSpec.setIsMetadataOnly(true);
}
Path metaDataPath = new Path(evRoot, EximUtil.METADATA_NAME);
EximUtil.createExportDump(metaDataPath.getFileSystem(conf), metaDataPath, qlMdTable, null, replicationSpec);
Path dataPath = new Path(evRoot, "data");
Iterable<String> files = ctm.getFiles();
if (files != null) {
// encoded filename/checksum of files, write into _files
FileSystem fs = dataPath.getFileSystem(conf);
Path filesPath = new Path(dataPath, EximUtil.FILES_NAME);
BufferedWriter fileListWriter = new BufferedWriter(new OutputStreamWriter(fs.create(filesPath)));
try {
for (String file : files) {
fileListWriter.write(file + "\n");
}
} finally {
fileListWriter.close();
}
}
(new DumpMetaData(evRoot, DUMPTYPE.EVENT_CREATE_TABLE, evid, evid, cmRoot)).write();
break;
}
case MessageFactory.ADD_PARTITION_EVENT:
{
AddPartitionMessage apm = md.getAddPartitionMessage(ev.getMessage());
LOG.info("Processing#{} ADD_PARTITION message : {}", ev.getEventId(), ev.getMessage());
Iterable<org.apache.hadoop.hive.metastore.api.Partition> ptns = apm.getPartitionObjs();
if ((ptns == null) || (!ptns.iterator().hasNext())) {
LOG.debug("Event#{} was an ADD_PTN_EVENT with no partitions");
break;
}
org.apache.hadoop.hive.metastore.api.Table tobj = apm.getTableObj();
if (tobj == null) {
LOG.debug("Event#{} was a ADD_PTN_EVENT with no table listed");
break;
}
final Table qlMdTable = new Table(tobj);
Iterable<Partition> qlPtns = Iterables.transform(ptns, new Function<org.apache.hadoop.hive.metastore.api.Partition, Partition>() {
@Nullable
@Override
public Partition apply(@Nullable org.apache.hadoop.hive.metastore.api.Partition input) {
if (input == null) {
return null;
}
try {
return new Partition(qlMdTable, input);
} catch (HiveException e) {
throw new IllegalArgumentException(e);
}
}
});
Path metaDataPath = new Path(evRoot, EximUtil.METADATA_NAME);
EximUtil.createExportDump(metaDataPath.getFileSystem(conf), metaDataPath, qlMdTable, qlPtns, replicationSpec);
Iterator<PartitionFiles> partitionFilesIter = apm.getPartitionFilesIter().iterator();
for (Partition qlPtn : qlPtns) {
PartitionFiles partitionFiles = partitionFilesIter.next();
Iterable<String> files = partitionFiles.getFiles();
if (files != null) {
// encoded filename/checksum of files, write into _files
Path ptnDataPath = new Path(evRoot, qlPtn.getName());
FileSystem fs = ptnDataPath.getFileSystem(conf);
Path filesPath = new Path(ptnDataPath, EximUtil.FILES_NAME);
BufferedWriter fileListWriter = new BufferedWriter(new OutputStreamWriter(fs.create(filesPath)));
try {
for (String file : files) {
fileListWriter.write(file + "\n");
}
} finally {
fileListWriter.close();
}
}
}
(new DumpMetaData(evRoot, DUMPTYPE.EVENT_ADD_PARTITION, evid, evid, cmRoot)).write();
break;
}
case MessageFactory.DROP_TABLE_EVENT:
{
LOG.info("Processing#{} DROP_TABLE message : {}", ev.getEventId(), ev.getMessage());
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_DROP_TABLE, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
break;
}
case MessageFactory.DROP_PARTITION_EVENT:
{
LOG.info("Processing#{} DROP_PARTITION message : {}", ev.getEventId(), ev.getMessage());
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_DROP_PARTITION, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
break;
}
case MessageFactory.ALTER_TABLE_EVENT:
{
LOG.info("Processing#{} ALTER_TABLE message : {}", ev.getEventId(), ev.getMessage());
AlterTableMessage atm = md.getAlterTableMessage(ev.getMessage());
org.apache.hadoop.hive.metastore.api.Table tobjBefore = atm.getTableObjBefore();
org.apache.hadoop.hive.metastore.api.Table tobjAfter = atm.getTableObjAfter();
if (tobjBefore.getDbName().equals(tobjAfter.getDbName()) && tobjBefore.getTableName().equals(tobjAfter.getTableName())) {
// regular alter scenario
replicationSpec.setIsMetadataOnly(true);
Table qlMdTableAfter = new Table(tobjAfter);
Path metaDataPath = new Path(evRoot, EximUtil.METADATA_NAME);
EximUtil.createExportDump(metaDataPath.getFileSystem(conf), metaDataPath, qlMdTableAfter, null, replicationSpec);
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_ALTER_TABLE, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
} else {
// rename scenario
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_RENAME_TABLE, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
}
break;
}
case MessageFactory.ALTER_PARTITION_EVENT:
{
LOG.info("Processing#{} ALTER_PARTITION message : {}", ev.getEventId(), ev.getMessage());
AlterPartitionMessage apm = md.getAlterPartitionMessage(ev.getMessage());
org.apache.hadoop.hive.metastore.api.Table tblObj = apm.getTableObj();
org.apache.hadoop.hive.metastore.api.Partition pobjBefore = apm.getPtnObjBefore();
org.apache.hadoop.hive.metastore.api.Partition pobjAfter = apm.getPtnObjAfter();
boolean renameScenario = false;
Iterator<String> beforeValIter = pobjBefore.getValuesIterator();
Iterator<String> afterValIter = pobjAfter.getValuesIterator();
for (; beforeValIter.hasNext(); ) {
if (!beforeValIter.next().equals(afterValIter.next())) {
renameScenario = true;
break;
}
}
if (!renameScenario) {
// regular partition alter
replicationSpec.setIsMetadataOnly(true);
Table qlMdTable = new Table(tblObj);
List<Partition> qlPtns = new ArrayList<Partition>();
qlPtns.add(new Partition(qlMdTable, pobjAfter));
Path metaDataPath = new Path(evRoot, EximUtil.METADATA_NAME);
EximUtil.createExportDump(metaDataPath.getFileSystem(conf), metaDataPath, qlMdTable, qlPtns, replicationSpec);
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_ALTER_PARTITION, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
break;
} else {
// rename scenario
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_RENAME_PARTITION, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
break;
}
}
case MessageFactory.INSERT_EVENT:
{
InsertMessage insertMsg = md.getInsertMessage(ev.getMessage());
String dbName = insertMsg.getDB();
String tblName = insertMsg.getTable();
org.apache.hadoop.hive.metastore.api.Table tobj = db.getMSC().getTable(dbName, tblName);
Table qlMdTable = new Table(tobj);
Map<String, String> partSpec = insertMsg.getPartitionKeyValues();
List<Partition> qlPtns = null;
if (qlMdTable.isPartitioned() && !partSpec.isEmpty()) {
qlPtns = Arrays.asList(db.getPartition(qlMdTable, partSpec, false));
}
Path metaDataPath = new Path(evRoot, EximUtil.METADATA_NAME);
// Mark the replication type as insert into to avoid overwrite while import
replicationSpec.setIsInsert(true);
EximUtil.createExportDump(metaDataPath.getFileSystem(conf), metaDataPath, qlMdTable, qlPtns, replicationSpec);
Iterable<String> files = insertMsg.getFiles();
if (files != null) {
// encoded filename/checksum of files, write into _files
Path dataPath = new Path(evRoot, EximUtil.DATA_PATH_NAME);
Path filesPath = new Path(dataPath, EximUtil.FILES_NAME);
FileSystem fs = dataPath.getFileSystem(conf);
BufferedWriter fileListWriter = new BufferedWriter(new OutputStreamWriter(fs.create(filesPath)));
try {
for (String file : files) {
fileListWriter.write(file + "\n");
}
} finally {
fileListWriter.close();
}
}
LOG.info("Processing#{} INSERT message : {}", ev.getEventId(), ev.getMessage());
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_INSERT, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
break;
}
// TODO : handle other event types
default:
LOG.info("Dummy processing#{} message : {}", ev.getEventId(), ev.getMessage());
DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_UNKNOWN, evid, evid, cmRoot);
dmd.setPayload(ev.getMessage());
dmd.write();
break;
}
}
use of org.apache.hadoop.hive.metastore.messaging.AlterTableMessage in project hive by apache.
the class TestDbNotificationListener method alterTable.
@Test
public void alterTable() throws Exception {
String defaultDbName = "default";
String tblName = "altertabletbl";
String tblOwner = "me";
String serdeLocation = testTempDir;
FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
FieldSchema col2 = new FieldSchema("col2", "int", "no comment");
List<FieldSchema> cols = new ArrayList<FieldSchema>();
cols.add(col1);
SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, new ArrayList<FieldSchema>(), emptyParameters, null, null, null);
// Event 1
msClient.createTable(table);
cols.add(col2);
table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, new ArrayList<FieldSchema>(), emptyParameters, null, null, null);
// Event 2
msClient.alter_table(defaultDbName, tblName, table);
// Get notifications from metastore
NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
assertEquals(2, rsp.getEventsSize());
NotificationEvent event = rsp.getEvents().get(1);
assertEquals(firstEventId + 2, event.getEventId());
assertTrue(event.getEventTime() >= startTime);
assertEquals(EventType.ALTER_TABLE.toString(), event.getEventType());
assertEquals(defaultDbName, event.getDbName());
assertEquals(tblName, event.getTableName());
AlterTableMessage alterTableMessage = md.getAlterTableMessage(event.getMessage());
assertEquals(table, alterTableMessage.getTableObjAfter());
assertEquals(TableType.MANAGED_TABLE.toString(), alterTableMessage.getTableType());
// Verify the eventID was passed to the non-transactional listener
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_TABLE, firstEventId + 2);
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
// When hive.metastore.transactional.event.listeners is set,
// a failed event should not create a new notification
DummyRawStoreFailEvent.setEventSucceed(false);
try {
msClient.alter_table(defaultDbName, tblName, table);
fail("Error: alter table should've failed");
} catch (Exception ex) {
// expected
}
rsp = msClient.getNextNotification(firstEventId, 0, null);
assertEquals(2, rsp.getEventsSize());
testEventCounts(defaultDbName, firstEventId, null, null, 2);
}
use of org.apache.hadoop.hive.metastore.messaging.AlterTableMessage in project hive by apache.
the class RenameTableHandler method handle.
@Override
public List<Task<?>> handle(Context context) throws SemanticException {
AlterTableMessage msg = deserializer.getAlterTableMessage(context.dmd.getPayload());
try {
Table tableObjBefore = msg.getTableObjBefore();
Table tableObjAfter = msg.getTableObjAfter();
String oldDbName = tableObjBefore.getDbName();
String newDbName = tableObjAfter.getDbName();
if (!context.isDbNameEmpty()) {
// newDbName must be the same
if (!oldDbName.equalsIgnoreCase(newDbName)) {
throw new SemanticException("Cannot replicate an event renaming a table across" + " databases into a db level load " + oldDbName + "->" + newDbName);
} else {
// both were the same, and can be replaced by the new db we're loading into.
oldDbName = context.dbName;
newDbName = context.dbName;
}
}
TableName oldName = TableName.fromString(tableObjBefore.getTableName(), null, oldDbName);
TableName newName = TableName.fromString(tableObjAfter.getTableName(), null, newDbName);
ReplicationSpec replicationSpec = context.eventOnlyReplicationSpec();
AlterTableRenameDesc renameTableDesc = new AlterTableRenameDesc(oldName, replicationSpec, false, newName.getNotEmptyDbTable());
renameTableDesc.setWriteId(msg.getWriteId());
Task<DDLWork> renameTableTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, renameTableDesc, true, context.getDumpDirectory(), context.getMetricCollector()), context.hiveConf);
context.log.debug("Added rename table task : {}:{}->{}", renameTableTask.getId(), oldName.getNotEmptyDbTable(), newName.getNotEmptyDbTable());
// oldDbName and newDbName *will* be the same if we're here
updatedMetadata.set(context.dmd.getEventTo().toString(), newDbName, tableObjAfter.getTableName(), null);
// if so. If that should ever change, this will need reworking.
return ReplUtils.addChildTask(renameTableTask);
} catch (Exception e) {
throw (e instanceof SemanticException) ? (SemanticException) e : new SemanticException("Error reading message members", e);
}
}
Aggregations