Examples with TableSnapshot org.apache.hadoop.hive.ql.io.AcidUtils.TableSnapshot used on opensource projects

Example 11 with TableSnapshot

use of org.apache.hadoop.hive.ql.io.AcidUtils.TableSnapshot in project hive by apache.

the class Hive method loadPartition.

/**
 * Load a directory into a Hive Table Partition - Alters existing content of
 * the partition with the contents of loadPath. - If the partition does not
 * exist - one is created - files in loadPath are moved into Hive. But the
 * directory itself is not removed.
 *
 * @param loadPath
 *          Directory containing files to load into Table
 * @param  tbl
 *          name of table to be loaded.
 * @param partSpec
 *          defines which partition needs to be loaded
 * @param loadFileType
 *          if REPLACE_ALL - replace files in the table,
 *          otherwise add files to table (KEEP_EXISTING, OVERWRITE_EXISTING)
 * @param inheritTableSpecs if true, on [re]creating the partition, take the
 *          location/inputformat/outputformat/serde details from table spec
 * @param isSrcLocal
 *          If the source directory is LOCAL
 * @param isAcidIUDoperation
 *          true if this is an ACID operation Insert/Update/Delete operation
 * @param resetStatistics
 *          if true, reset the statistics. If false, do not reset statistics.
 * @param writeId write ID allocated for the current load operation
 * @param stmtId statement ID of the current load statement
 * @param isInsertOverwrite
 * @return Partition object being loaded with data
 */
public Partition loadPartition(Path loadPath, Table tbl, Map<String, String> partSpec, LoadFileType loadFileType, boolean inheritTableSpecs, boolean inheritLocation, boolean isSkewedStoreAsSubdir, boolean isSrcLocal, boolean isAcidIUDoperation, boolean resetStatistics, Long writeId, int stmtId, boolean isInsertOverwrite, boolean isDirectInsert) throws HiveException {
    PerfLogger perfLogger = SessionState.getPerfLogger();
    perfLogger.perfLogBegin("MoveTask", PerfLogger.LOAD_PARTITION);
    // Get the partition object if it already exists
    Partition oldPart = getPartition(tbl, partSpec, false);
    boolean isTxnTable = AcidUtils.isTransactionalTable(tbl);
    // If config is set, table is not temporary and partition being inserted exists, capture
    // the list of files added. For not yet existing partitions (insert overwrite to new partition
    // or dynamic partition inserts), the add partition event will capture the list of files added.
    List<FileStatus> newFiles = null;
    if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
        newFiles = Collections.synchronizedList(new ArrayList<>());
    }
    Partition newTPart = loadPartitionInternal(loadPath, tbl, partSpec, oldPart, loadFileType, inheritTableSpecs, inheritLocation, isSkewedStoreAsSubdir, isSrcLocal, isAcidIUDoperation, resetStatistics, writeId, stmtId, isInsertOverwrite, isTxnTable, newFiles, isDirectInsert);
    AcidUtils.TableSnapshot tableSnapshot = isTxnTable ? getTableSnapshot(tbl, writeId) : null;
    if (tableSnapshot != null) {
        newTPart.getTPartition().setWriteId(tableSnapshot.getWriteId());
    }
    if (oldPart == null) {
        addPartitionToMetastore(newTPart, resetStatistics, tbl, tableSnapshot);
        // it should be done after partition is created.
        if (isTxnTable && (null != newFiles)) {
            addWriteNotificationLog(tbl, partSpec, newFiles, writeId, null);
        }
    } else {
        try {
            setStatsPropAndAlterPartition(resetStatistics, tbl, newTPart, tableSnapshot);
        } catch (TException e) {
            LOG.error("Error loading partitions", e);
            throw new HiveException(e);
        }
    }
    perfLogger.perfLogEnd("MoveTask", PerfLogger.LOAD_PARTITION);
    return newTPart;
}

Example 12 with TableSnapshot

use of org.apache.hadoop.hive.ql.io.AcidUtils.TableSnapshot in project hive by apache.

the class Hive method alterTable.

public void alterTable(String catName, String dbName, String tblName, Table newTbl, boolean cascade, EnvironmentContext environmentContext, boolean transactional, long replWriteId) throws HiveException {
    if (catName == null) {
        catName = getDefaultCatalog(conf);
    }
    try {
        // Remove the DDL_TIME so it gets refreshed
        if (newTbl.getParameters() != null) {
            newTbl.getParameters().remove(hive_metastoreConstants.DDL_TIME);
        }
        newTbl.checkValidity(conf);
        if (environmentContext == null) {
            environmentContext = new EnvironmentContext();
        }
        if (cascade) {
            environmentContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);
        }
        // Take a table snapshot and set it to newTbl.
        AcidUtils.TableSnapshot tableSnapshot = null;
        if (transactional) {
            if (replWriteId > 0) {
                // We need a valid writeId list for a transactional table modification. During
                // replication we do not have a valid writeId list which was used to modify the table
                // on the source. But we know for sure that the writeId associated with it was valid
                // then (otherwise modification would have failed on the source). So use a valid
                // transaction list with only that writeId.
                ValidWriteIdList writeIds = new ValidReaderWriteIdList(TableName.getDbTable(dbName, tblName), new long[0], new BitSet(), replWriteId);
                tableSnapshot = new TableSnapshot(replWriteId, writeIds.writeToString());
            } else {
                // Make sure we pass in the names, so we can get the correct snapshot for rename table.
                tableSnapshot = AcidUtils.getTableSnapshot(conf, newTbl, dbName, tblName, true);
            }
            if (tableSnapshot != null) {
                newTbl.getTTable().setWriteId(tableSnapshot.getWriteId());
            } else {
                LOG.warn("Cannot get a table snapshot for " + tblName);
            }
        }
        // Why is alter_partitions synchronized while this isn't?
        getMSC().alter_table(catName, dbName, tblName, newTbl.getTTable(), environmentContext, tableSnapshot == null ? null : tableSnapshot.getValidWriteIdList());
    } catch (MetaException e) {
        throw new HiveException("Unable to alter table. " + e.getMessage(), e);
    } catch (TException e) {
        throw new HiveException("Unable to alter table. " + e.getMessage(), e);
    }
}

Example 13 with TableSnapshot

use of org.apache.hadoop.hive.ql.io.AcidUtils.TableSnapshot in project hive by apache.

the class Hive method createTable.

// TODO: from here down dozens of methods do not support catalog. I got tired marking them.
/**
 * Creates the table with the given objects. It takes additional arguments for
 * primary keys and foreign keys associated with the table.
 *
 * @param tbl
 *          a table object
 * @param ifNotExists
 *          if true, ignore AlreadyExistsException
 * @param primaryKeys
 *          primary key columns associated with the table
 * @param foreignKeys
 *          foreign key columns associated with the table
 * @param uniqueConstraints
 *          UNIQUE constraints associated with the table
 * @param notNullConstraints
 *          NOT NULL constraints associated with the table
 * @param defaultConstraints
 *          DEFAULT constraints associated with the table
 * @param checkConstraints
 *          CHECK constraints associated with the table
 * @throws HiveException
 */
public void createTable(Table tbl, boolean ifNotExists, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys, List<SQLUniqueConstraint> uniqueConstraints, List<SQLNotNullConstraint> notNullConstraints, List<SQLDefaultConstraint> defaultConstraints, List<SQLCheckConstraint> checkConstraints) throws HiveException {
    try {
        if (org.apache.commons.lang3.StringUtils.isBlank(tbl.getDbName())) {
            tbl.setDbName(SessionState.get().getCurrentDatabase());
        }
        if (tbl.getCols().size() == 0 || tbl.getSd().getColsSize() == 0) {
            tbl.setFields(HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), tbl.getDeserializer()));
        }
        tbl.checkValidity(conf);
        if (tbl.getParameters() != null) {
            tbl.getParameters().remove(hive_metastoreConstants.DDL_TIME);
        }
        org.apache.hadoop.hive.metastore.api.Table tTbl = tbl.getTTable();
        PrincipalPrivilegeSet principalPrivs = new PrincipalPrivilegeSet();
        SessionState ss = SessionState.get();
        if (ss != null) {
            CreateTableAutomaticGrant grants = ss.getCreateTableGrants();
            if (grants != null) {
                principalPrivs.setUserPrivileges(grants.getUserGrants());
                principalPrivs.setGroupPrivileges(grants.getGroupGrants());
                principalPrivs.setRolePrivileges(grants.getRoleGrants());
                tTbl.setPrivileges(principalPrivs);
            }
            if (AcidUtils.isTransactionalTable(tbl)) {
                boolean createTableUseSuffix = HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX) || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
                if (createTableUseSuffix) {
                    tbl.setProperty(SOFT_DELETE_TABLE, Boolean.TRUE.toString());
                }
                tTbl.setTxnId(ss.getTxnMgr().getCurrentTxnId());
            }
        }
        // crafting one on the replica.
        if (tTbl.getWriteId() <= 0) {
            TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, true);
            if (tableSnapshot != null) {
                tTbl.setWriteId(tableSnapshot.getWriteId());
            }
        }
        if (primaryKeys == null && foreignKeys == null && uniqueConstraints == null && notNullConstraints == null && defaultConstraints == null && checkConstraints == null) {
            getMSC().createTable(tTbl);
        } else {
            getMSC().createTableWithConstraints(tTbl, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints);
        }
    } catch (AlreadyExistsException e) {
        if (!ifNotExists) {
            throw new HiveException(e);
        }
    } catch (Exception e) {
        throw new HiveException(e);
    }
}

Example 14 with TableSnapshot

use of org.apache.hadoop.hive.ql.io.AcidUtils.TableSnapshot in project hive by apache.

the class Hive method truncateTable.

/**
 * Truncates the table/partition as per specifications. Just trash the data files
 *
 * @param dbDotTableName
 *          name of the table
 * @throws HiveException
 */
public void truncateTable(String dbDotTableName, Map<String, String> partSpec, Long writeId) throws HiveException {
    try {
        Table table = getTable(dbDotTableName, true);
        AcidUtils.TableSnapshot snapshot = null;
        if (AcidUtils.isTransactionalTable(table)) {
            if (writeId <= 0) {
                snapshot = AcidUtils.getTableSnapshot(conf, table, true);
            } else {
                String fullTableName = getFullTableName(table.getDbName(), table.getTableName());
                ValidWriteIdList writeIdList = getMSC().getValidWriteIds(fullTableName, writeId);
                snapshot = new TableSnapshot(writeId, writeIdList.writeToString());
            }
        }
        // TODO: APIs with catalog names
        List<String> partNames = ((null == partSpec) ? null : getPartitionNames(table.getDbName(), table.getTableName(), partSpec, (short) -1));
        if (snapshot == null) {
            getMSC().truncateTable(table.getDbName(), table.getTableName(), partNames);
        } else {
            boolean truncateUseBase = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE) || HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
            getMSC().truncateTable(table.getDbName(), table.getTableName(), partNames, snapshot.getValidWriteIdList(), snapshot.getWriteId(), !truncateUseBase);
        }
    } catch (Exception e) {
        throw new HiveException(e);
    }
}