Search in sources :

Example 11 with HoodieHiveSyncException

use of org.apache.hudi.hive.HoodieHiveSyncException in project hudi by apache.

the class HMSDDLExecutor method createDatabase.

@Override
public void createDatabase(String databaseName) {
    try {
        Database database = new Database(databaseName, "automatically created by hoodie", null, null);
        client.createDatabase(database);
    } catch (Exception e) {
        LOG.error("Failed to create database " + databaseName, e);
        throw new HoodieHiveSyncException("Failed to create database " + databaseName, e);
    }
}
Also used : Database(org.apache.hadoop.hive.metastore.api.Database) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) TException(org.apache.thrift.TException) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException)

Example 12 with HoodieHiveSyncException

use of org.apache.hudi.hive.HoodieHiveSyncException in project hudi by apache.

the class HMSDDLExecutor method dropPartitionsToTable.

@Override
public void dropPartitionsToTable(String tableName, List<String> partitionsToDrop) {
    if (partitionsToDrop.isEmpty()) {
        LOG.info("No partitions to drop for " + tableName);
        return;
    }
    LOG.info("Drop partitions " + partitionsToDrop.size() + " on " + tableName);
    try {
        for (String dropPartition : partitionsToDrop) {
            String partitionClause = HivePartitionUtil.getPartitionClauseForDrop(dropPartition, partitionValueExtractor, syncConfig);
            client.dropPartition(syncConfig.databaseName, tableName, partitionClause, false);
            LOG.info("Drop partition " + dropPartition + " on " + tableName);
        }
    } catch (TException e) {
        LOG.error(syncConfig.databaseName + "." + tableName + " drop partition failed", e);
        throw new HoodieHiveSyncException(syncConfig.databaseName + "." + tableName + " drop partition failed", e);
    }
}
Also used : TException(org.apache.thrift.TException) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException)

Example 13 with HoodieHiveSyncException

use of org.apache.hudi.hive.HoodieHiveSyncException in project hudi by apache.

the class HMSDDLExecutor method updateTableComments.

@Override
public void updateTableComments(String tableName, Map<String, ImmutablePair<String, String>> alterSchema) {
    try {
        Table table = client.getTable(syncConfig.databaseName, tableName);
        StorageDescriptor sd = new StorageDescriptor(table.getSd());
        for (FieldSchema fieldSchema : sd.getCols()) {
            if (alterSchema.containsKey(fieldSchema.getName())) {
                String comment = alterSchema.get(fieldSchema.getName()).getRight();
                fieldSchema.setComment(comment);
            }
        }
        table.setSd(sd);
        EnvironmentContext environmentContext = new EnvironmentContext();
        client.alter_table_with_environmentContext(syncConfig.databaseName, tableName, table, environmentContext);
        sd.clear();
    } catch (Exception e) {
        LOG.error("Failed to update table comments for " + tableName, e);
        throw new HoodieHiveSyncException("Failed to update table comments for " + tableName, e);
    }
}
Also used : EnvironmentContext(org.apache.hadoop.hive.metastore.api.EnvironmentContext) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) TException(org.apache.thrift.TException) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException)

Example 14 with HoodieHiveSyncException

use of org.apache.hudi.hive.HoodieHiveSyncException in project hudi by apache.

the class HMSDDLExecutor method createTable.

@Override
public void createTable(String tableName, MessageType storageSchema, String inputFormatClass, String outputFormatClass, String serdeClass, Map<String, String> serdeProperties, Map<String, String> tableProperties) {
    try {
        LinkedHashMap<String, String> mapSchema = HiveSchemaUtil.parquetSchemaToMapSchema(storageSchema, syncConfig.supportTimestamp, false);
        List<FieldSchema> fieldSchema = HiveSchemaUtil.convertMapSchemaToHiveFieldSchema(mapSchema, syncConfig);
        List<FieldSchema> partitionSchema = syncConfig.partitionFields.stream().map(partitionKey -> {
            String partitionKeyType = HiveSchemaUtil.getPartitionKeyType(mapSchema, partitionKey);
            return new FieldSchema(partitionKey, partitionKeyType.toLowerCase(), "");
        }).collect(Collectors.toList());
        Table newTb = new Table();
        newTb.setDbName(syncConfig.databaseName);
        newTb.setTableName(tableName);
        newTb.setOwner(UserGroupInformation.getCurrentUser().getShortUserName());
        newTb.setCreateTime((int) System.currentTimeMillis());
        StorageDescriptor storageDescriptor = new StorageDescriptor();
        storageDescriptor.setCols(fieldSchema);
        storageDescriptor.setInputFormat(inputFormatClass);
        storageDescriptor.setOutputFormat(outputFormatClass);
        storageDescriptor.setLocation(syncConfig.basePath);
        serdeProperties.put("serialization.format", "1");
        storageDescriptor.setSerdeInfo(new SerDeInfo(null, serdeClass, serdeProperties));
        newTb.setSd(storageDescriptor);
        newTb.setPartitionKeys(partitionSchema);
        if (!syncConfig.createManagedTable) {
            newTb.putToParameters("EXTERNAL", "TRUE");
        }
        for (Map.Entry<String, String> entry : tableProperties.entrySet()) {
            newTb.putToParameters(entry.getKey(), entry.getValue());
        }
        newTb.setTableType(TableType.EXTERNAL_TABLE.toString());
        client.createTable(newTb);
    } catch (Exception e) {
        LOG.error("failed to create table " + tableName, e);
        throw new HoodieHiveSyncException("failed to create table " + tableName, e);
    }
}
Also used : ImmutablePair(org.apache.hudi.common.util.collection.ImmutablePair) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) FileSystem(org.apache.hadoop.fs.FileSystem) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) Partition(org.apache.hadoop.hive.metastore.api.Partition) LinkedHashMap(java.util.LinkedHashMap) HiveSchemaUtil(org.apache.hudi.hive.util.HiveSchemaUtil) Logger(org.apache.log4j.Logger) StatsSetupConst(org.apache.hadoop.hive.common.StatsSetupConst) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) HivePartitionUtil(org.apache.hudi.hive.util.HivePartitionUtil) PartitionValueExtractor(org.apache.hudi.hive.PartitionValueExtractor) Hive(org.apache.hadoop.hive.ql.metadata.Hive) HiveSyncConfig(org.apache.hudi.hive.HiveSyncConfig) HiveConf(org.apache.hadoop.hive.conf.HiveConf) EnvironmentContext(org.apache.hadoop.hive.metastore.api.EnvironmentContext) TException(org.apache.thrift.TException) StorageSchemes(org.apache.hudi.common.fs.StorageSchemes) Collectors(java.util.stream.Collectors) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) MessageType(org.apache.parquet.schema.MessageType) List(java.util.List) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) IMetaStoreClient(org.apache.hadoop.hive.metastore.IMetaStoreClient) TableType(org.apache.hadoop.hive.metastore.TableType) LogManager(org.apache.log4j.LogManager) Database(org.apache.hadoop.hive.metastore.api.Database) FSUtils(org.apache.hudi.common.fs.FSUtils) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) TException(org.apache.thrift.TException) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException)

Example 15 with HoodieHiveSyncException

use of org.apache.hudi.hive.HoodieHiveSyncException in project hudi by apache.

the class HiveQueryDDLExecutor method getTableSchema.

// TODO Duplicating it here from HMSDLExecutor as HiveQueryQL has no way of doing it on its own currently. Need to refactor it
@Override
public Map<String, String> getTableSchema(String tableName) {
    try {
        // HiveMetastoreClient returns partition keys separate from Columns, hence get both and merge to
        // get the Schema of the table.
        final long start = System.currentTimeMillis();
        Table table = metaStoreClient.getTable(config.databaseName, tableName);
        Map<String, String> partitionKeysMap = table.getPartitionKeys().stream().collect(Collectors.toMap(FieldSchema::getName, f -> f.getType().toUpperCase()));
        Map<String, String> columnsMap = table.getSd().getCols().stream().collect(Collectors.toMap(FieldSchema::getName, f -> f.getType().toUpperCase()));
        Map<String, String> schema = new HashMap<>();
        schema.putAll(columnsMap);
        schema.putAll(partitionKeysMap);
        final long end = System.currentTimeMillis();
        LOG.info(String.format("Time taken to getTableSchema: %s ms", (end - start)));
        return schema;
    } catch (Exception e) {
        throw new HoodieHiveSyncException("Failed to get table schema for : " + tableName, e);
    }
}
Also used : MetaException(org.apache.hadoop.hive.metastore.api.MetaException) FileSystem(org.apache.hadoop.fs.FileSystem) HashMap(java.util.HashMap) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Map(java.util.Map) HivePartitionUtil(org.apache.hudi.hive.util.HivePartitionUtil) Hive(org.apache.hadoop.hive.ql.metadata.Hive) HiveSyncConfig(org.apache.hudi.hive.HiveSyncConfig) HiveConf(org.apache.hadoop.hive.conf.HiveConf) Driver(org.apache.hadoop.hive.ql.Driver) IOException(java.io.IOException) SessionState(org.apache.hadoop.hive.ql.session.SessionState) Collectors(java.util.stream.Collectors) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) CommandProcessorResponse(org.apache.hadoop.hive.ql.processors.CommandProcessorResponse) List(java.util.List) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) IMetaStoreClient(org.apache.hadoop.hive.metastore.IMetaStoreClient) LogManager(org.apache.log4j.LogManager) Collections(java.util.Collections) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) Table(org.apache.hadoop.hive.metastore.api.Table) HashMap(java.util.HashMap) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) IOException(java.io.IOException) HoodieHiveSyncException(org.apache.hudi.hive.HoodieHiveSyncException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException)

Aggregations

HoodieHiveSyncException (org.apache.hudi.hive.HoodieHiveSyncException)18 MetaException (org.apache.hadoop.hive.metastore.api.MetaException)10 HiveException (org.apache.hadoop.hive.ql.metadata.HiveException)10 TException (org.apache.thrift.TException)8 IOException (java.io.IOException)7 HashMap (java.util.HashMap)7 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)7 Table (org.apache.hadoop.hive.metastore.api.Table)7 Map (java.util.Map)6 EnvironmentContext (org.apache.hadoop.hive.metastore.api.EnvironmentContext)6 StorageDescriptor (org.apache.hadoop.hive.metastore.api.StorageDescriptor)6 LinkedHashMap (java.util.LinkedHashMap)5 List (java.util.List)5 Collectors (java.util.stream.Collectors)5 FileSystem (org.apache.hadoop.fs.FileSystem)5 HiveConf (org.apache.hadoop.hive.conf.HiveConf)5 IMetaStoreClient (org.apache.hadoop.hive.metastore.IMetaStoreClient)5 Database (org.apache.hadoop.hive.metastore.api.Database)5 Hive (org.apache.hadoop.hive.ql.metadata.Hive)5 UserGroupInformation (org.apache.hadoop.security.UserGroupInformation)5