Examples with ReplicationSpec org.apache.hadoop.hive.ql.parse.ReplicationSpec used on opensource projects

Example 6 with ReplicationSpec

use of org.apache.hadoop.hive.ql.parse.ReplicationSpec in project hive by apache.

the class ReplTxnTask method execute.

@Override
public int execute() {
    String replPolicy = work.getReplPolicy();
    String tableName = work.getTableName();
    ReplicationSpec replicationSpec = work.getReplicationSpec();
    if ((tableName != null) && (replicationSpec != null)) {
        try {
            Database database = Hive.get().getDatabase(work.getDbName());
            if (!replicationSpec.allowReplacementInto(database.getParameters())) {
                // if the event is already replayed, then no need to replay it again.
                LOG.debug("ReplTxnTask: Event is skipped as it is already replayed. Event Id: " + replicationSpec.getReplicationState() + "Event Type: " + work.getOperationType());
                return 0;
            }
        } catch (HiveException e1) {
            LOG.error("Get database failed with exception " + e1.getMessage());
            return 1;
        }
    }
    try {
        HiveTxnManager txnManager = context.getHiveTxnManager();
        String user = UserGroupInformation.getCurrentUser().getUserName();
        switch(work.getOperationType()) {
            case REPL_OPEN_TXN:
                List<Long> txnIds = txnManager.replOpenTxn(replPolicy, work.getTxnIds(), user);
                assert txnIds.size() == work.getTxnIds().size();
                LOG.info("Replayed OpenTxn Event for policy " + replPolicy + " with srcTxn " + work.getTxnIds().toString() + " and target txn id " + txnIds.toString());
                return 0;
            case REPL_ABORT_TXN:
                for (long txnId : work.getTxnIds()) {
                    txnManager.replRollbackTxn(replPolicy, txnId);
                    LOG.info("Replayed AbortTxn Event for policy " + replPolicy + " with srcTxn " + txnId);
                }
                return 0;
            case REPL_COMMIT_TXN:
                // Currently only one commit txn per event is supported.
                assert (work.getTxnIds().size() == 1);
                long txnId = work.getTxnIds().get(0);
                CommitTxnRequest commitTxnRequest = new CommitTxnRequest(txnId);
                commitTxnRequest.setReplPolicy(work.getReplPolicy());
                commitTxnRequest.setWriteEventInfos(work.getWriteEventInfos());
                commitTxnRequest.setTxn_type(TxnType.REPL_CREATED);
                txnManager.replCommitTxn(commitTxnRequest);
                LOG.info("Replayed CommitTxn Event for replPolicy: " + replPolicy + " with srcTxn: " + txnId + "WriteEventInfos: " + work.getWriteEventInfos());
                return 0;
            case REPL_ALLOC_WRITE_ID:
                assert work.getTxnToWriteIdList() != null;
                String dbName = work.getDbName();
                List<TxnToWriteId> txnToWriteIdList = work.getTxnToWriteIdList();
                txnManager.replAllocateTableWriteIdsBatch(dbName, tableName, replPolicy, txnToWriteIdList);
                LOG.info("Replayed alloc write Id Event for repl policy: " + replPolicy + " db Name : " + dbName + " txnToWriteIdList: " + txnToWriteIdList.toString() + " table name: " + tableName);
                return 0;
            case REPL_WRITEID_STATE:
                txnManager.replTableWriteIdState(work.getValidWriteIdList(), work.getDbName(), tableName, work.getPartNames());
                LOG.info("Replicated WriteId state for DbName: " + work.getDbName() + " TableName: " + tableName + " ValidWriteIdList: " + work.getValidWriteIdList());
                return 0;
            default:
                LOG.error("Operation Type " + work.getOperationType() + " is not supported ");
                return 1;
        }
    } catch (Exception e) {
        console.printError("Failed with exception " + e.getMessage(), "\n" + StringUtils.stringifyException(e));
        setException(e);
        LOG.error("ReplTxnTask failed", e);
        return ReplUtils.handleException(true, e, work.getDumpDirectory(), work.getMetricCollector(), getName(), conf);
    }
}

Example 7 with ReplicationSpec

use of org.apache.hadoop.hive.ql.parse.ReplicationSpec in project hive by apache.

the class CreateTableOperation method createTableReplaceMode.

private void createTableReplaceMode(Table tbl, boolean replDataLocationChanged) throws HiveException {
    ReplicationSpec replicationSpec = desc.getReplicationSpec();
    Long writeId = 0L;
    EnvironmentContext environmentContext = null;
    if (replicationSpec != null && replicationSpec.isInReplicationScope()) {
        writeId = desc.getReplWriteId();
        // In case of replication statistics is obtained from the source, so do not update those
        // on replica.
        environmentContext = new EnvironmentContext();
        environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
    }
    // environment context to notify Metastore to update location of all partitions and delete old directory.
    if (replDataLocationChanged) {
        environmentContext = ReplUtils.setReplDataLocationChangedFlag(environmentContext);
    }
    // replace-mode creates are really alters using CreateTableDesc.
    context.getDb().alterTable(tbl.getCatName(), tbl.getDbName(), tbl.getTableName(), tbl, false, environmentContext, true, writeId);
}

Example 8 with ReplicationSpec

use of org.apache.hadoop.hive.ql.parse.ReplicationSpec in project hive by apache.

the class AbstractDropPartitionAnalyzer method analyzeCommand.

@Override
protected void analyzeCommand(TableName tableName, Map<String, String> partitionSpec, ASTNode command) throws SemanticException {
    boolean ifExists = (command.getFirstChildWithType(HiveParser.TOK_IFEXISTS) != null) || HiveConf.getBoolVar(conf, ConfVars.DROP_IGNORES_NON_EXISTENT);
    // If the drop has to fail on non-existent partitions, we cannot batch expressions.
    // That is because we actually have to check each separate expression for existence.
    // We could do a small optimization for the case where expr has all columns and all
    // operators are equality, if we assume those would always match one partition (which
    // may not be true with legacy, non-normalized column values). This is probably a
    // popular case but that's kinda hacky. Let's not do it for now.
    boolean canGroupExprs = ifExists;
    boolean mustPurge = (command.getFirstChildWithType(HiveParser.KW_PURGE) != null);
    ReplicationSpec replicationSpec = new ReplicationSpec(command);
    Table table = null;
    try {
        table = getTable(tableName);
    } catch (SemanticException se) {
        if (replicationSpec.isInReplicationScope() && ((se.getCause() instanceof InvalidTableException) || (se.getMessage().contains(ErrorMsg.INVALID_TABLE.getMsg())))) {
            // We just return in that case, no drop needed.
            return;
        // TODO : the contains message check is fragile, we should refactor SemanticException to be
        // queriable for error code, and not simply have a message
        // NOTE : IF_EXISTS might also want to invoke this, but there's a good possibility
        // that IF_EXISTS is stricter about table existence, and applies only to the ptn.
        // Therefore, ignoring IF_EXISTS here.
        } else {
            throw se;
        }
    }
    validateAlterTableType(table, AlterTableType.DROPPARTITION, expectView());
    Map<Integer, List<ExprNodeGenericFuncDesc>> partitionSpecs = ParseUtils.getFullPartitionSpecs(command, table, conf, canGroupExprs);
    if (partitionSpecs.isEmpty()) {
        // nothing to do
        return;
    }
    ReadEntity re = new ReadEntity(table);
    re.noLockNeeded();
    inputs.add(re);
    boolean dropPartUseBase = HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_DROP_PARTITION_USE_BASE) || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED) && AcidUtils.isTransactionalTable(table);
    addTableDropPartsOutputs(table, partitionSpecs.values(), !ifExists, dropPartUseBase);
    AlterTableDropPartitionDesc desc = new AlterTableDropPartitionDesc(tableName, partitionSpecs, mustPurge, replicationSpec, !dropPartUseBase, table);
    if (desc.mayNeedWriteId()) {
        setAcidDdlDesc(desc);
    }
    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
}

Example 9 with ReplicationSpec

use of org.apache.hadoop.hive.ql.parse.ReplicationSpec in project hive by apache.

the class AlterTableRenamePartitionOperation method execute.

@Override
public int execute() throws HiveException {
    String tableName = desc.getTableName();
    Map<String, String> oldPartSpec = desc.getOldPartSpec();
    ReplicationSpec replicationSpec = desc.getReplicationSpec();
    if (!AlterTableUtils.allowOperationInReplicationScope(context.getDb(), tableName, oldPartSpec, replicationSpec)) {
        // no rename, the table is missing either due to drop/rename which follows the current rename.
        // or the existing table is newer than our update.
        LOG.debug("DDLTask: Rename Partition is skipped as table {} / partition {} is newer than update", tableName, FileUtils.makePartName(new ArrayList<>(oldPartSpec.keySet()), new ArrayList<>(oldPartSpec.values())));
        return 0;
    }
    if (Utils.isBootstrapDumpInProgress(context.getDb(), HiveTableName.of(tableName).getDb())) {
        LOG.error("DDLTask: Rename Partition not allowed as bootstrap dump in progress");
        throw new HiveException("Rename Partition: Not allowed as bootstrap dump in progress");
    }
    Table tbl = context.getDb().getTable(tableName);
    Partition oldPart = context.getDb().getPartition(tbl, oldPartSpec, false);
    if (oldPart == null) {
        String partName = FileUtils.makePartName(new ArrayList<String>(oldPartSpec.keySet()), new ArrayList<String>(oldPartSpec.values()));
        throw new HiveException("Rename partition: source partition [" + partName + "] does not exist.");
    }
    Partition part = context.getDb().getPartition(tbl, oldPartSpec, false);
    part.setValues(desc.getNewPartSpec());
    long writeId = desc.getWriteId();
    context.getDb().renamePartition(tbl, oldPartSpec, part, writeId);
    Partition newPart = context.getDb().getPartition(tbl, desc.getNewPartSpec(), false);
    context.getWork().getInputs().add(new ReadEntity(oldPart));
    // We've already obtained a lock on the table, don't lock the partition too
    DDLUtils.addIfAbsentByName(new WriteEntity(newPart, WriteEntity.WriteType.DDL_NO_LOCK), context);
    return 0;
}

Example 10 with ReplicationSpec

use of org.apache.hadoop.hive.ql.parse.ReplicationSpec in project hive by apache.

the class AlterTableDropPartitionOperation method execute.

@Override
public int execute() throws HiveException {
    // We need to fetch the table before it is dropped so that it can be passed to post-execution hook
    Table table = null;
    try {
        table = context.getDb().getTable(desc.getTableName());
    } catch (InvalidTableException e) {
    // drop table is idempotent
    }
    ReplicationSpec replicationSpec = desc.getReplicationSpec();
    if (replicationSpec.isInReplicationScope()) {
        dropPartitionForReplication(table, replicationSpec);
    } else {
        dropPartitions(false);
    }
    return 0;
}