Examples with WriteEventInfo org.apache.hadoop.hive.metastore.api.WriteEventInfo used on opensource projects

Example 6 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class TestGetAllWriteEventInfo method testGetByWrongDB.

@Test
public void testGetByWrongDB() throws Exception {
    GetAllWriteEventInfoRequest req = new GetAllWriteEventInfoRequest();
    req.setTxnId(TXN_ID);
    req.setDbName("wrong_db");
    List<WriteEventInfo> writeEventInfoList = client.getAllWriteEventInfo(req);
    Assert.assertTrue(writeEventInfoList.isEmpty());
}

Example 7 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class ObjectStore method getAllWriteEventInfo.

@Override
public List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException {
    List<WriteEventInfo> writeEventInfoList = null;
    boolean commited = false;
    Query query = null;
    try {
        openTransaction();
        List<String> parameterVals = new ArrayList<>();
        StringBuilder filterBuilder = new StringBuilder(" txnId == " + Long.toString(txnId));
        if (dbName != null && !"*".equals(dbName)) {
            // * means get all database, so no need to add filter
            appendSimpleCondition(filterBuilder, "database", new String[] { dbName }, parameterVals);
        }
        if (tableName != null && !"*".equals(tableName)) {
            appendSimpleCondition(filterBuilder, "table", new String[] { tableName }, parameterVals);
        }
        query = pm.newQuery(MTxnWriteNotificationLog.class, filterBuilder.toString());
        query.setOrdering("database,table ascending");
        List<MTxnWriteNotificationLog> mplans = (List<MTxnWriteNotificationLog>) query.executeWithArray(parameterVals.toArray(new String[0]));
        pm.retrieveAll(mplans);
        commited = commitTransaction();
        if (mplans != null && mplans.size() > 0) {
            writeEventInfoList = Lists.newArrayList();
            for (MTxnWriteNotificationLog mplan : mplans) {
                WriteEventInfo writeEventInfo = new WriteEventInfo(mplan.getWriteId(), mplan.getDatabase(), mplan.getTable(), mplan.getFiles());
                writeEventInfo.setPartition(mplan.getPartition());
                writeEventInfo.setPartitionObj(mplan.getPartObject());
                writeEventInfo.setTableObj(mplan.getTableObject());
                writeEventInfoList.add(writeEventInfo);
            }
        }
    } finally {
        rollbackAndCleanup(commited, query);
    }
    return writeEventInfoList;
}

Example 8 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class CommitTxnHandler method handle.

@Override
public List<Task<?>> handle(Context context) throws SemanticException {
    if (!AcidUtils.isAcidEnabled(context.hiveConf)) {
        context.log.error("Cannot load transaction events as acid is not enabled");
        throw new SemanticException("Cannot load transaction events as acid is not enabled");
    }
    CommitTxnMessage msg = deserializer.getCommitTxnMessage(context.dmd.getPayload());
    int numEntry = (msg.getTables() == null ? 0 : msg.getTables().size());
    List<Task<?>> tasks = new ArrayList<>();
    String dbName = context.dbName;
    String tableNamePrev = null;
    String tblName = null;
    ReplTxnWork work = new ReplTxnWork(HiveUtils.getReplPolicy(context.dbName), context.dbName, null, msg.getTxnId(), ReplTxnWork.OperationType.REPL_COMMIT_TXN, context.eventOnlyReplicationSpec(), context.getDumpDirectory(), context.getMetricCollector());
    if (numEntry > 0) {
        context.log.debug("Commit txn handler for txnid " + msg.getTxnId() + " databases : " + msg.getDatabases() + " tables : " + msg.getTables() + " partitions : " + msg.getPartitions() + " files : " + msg.getFilesList() + " write ids : " + msg.getWriteIds());
    }
    for (int idx = 0; idx < numEntry; idx++) {
        String actualTblName = msg.getTables().get(idx);
        String actualDBName = msg.getDatabases().get(idx);
        String completeName = Table.getCompleteName(actualDBName, actualTblName);
        // grouped together in commit txn message.
        if (tableNamePrev == null || !(completeName.equals(tableNamePrev))) {
            // The data location is created by source, so the location should be formed based on the table name in msg.
            Path location = HiveUtils.getDumpPath(new Path(context.location), actualDBName, actualTblName);
            tblName = actualTblName;
            // for warehouse level dump, use db name from write event
            dbName = (context.isDbNameEmpty() ? actualDBName : context.dbName);
            Context currentContext = new Context(context, dbName, context.getDumpDirectory(), context.getMetricCollector());
            currentContext.setLocation(location.toUri().toString());
            // Piggybacking in Import logic for now
            TableHandler tableHandler = new TableHandler();
            tasks.addAll((tableHandler.handle(currentContext)));
            readEntitySet.addAll(tableHandler.readEntities());
            writeEntitySet.addAll(tableHandler.writeEntities());
            getUpdatedMetadata().copyUpdatedMetadata(tableHandler.getUpdatedMetadata());
            tableNamePrev = completeName;
        }
        try {
            WriteEventInfo writeEventInfo = new WriteEventInfo(msg.getWriteIds().get(idx), dbName, tblName, msg.getFiles(idx));
            if (msg.getPartitions().get(idx) != null && !msg.getPartitions().get(idx).isEmpty()) {
                writeEventInfo.setPartition(msg.getPartitions().get(idx));
            }
            work.addWriteEventInfo(writeEventInfo);
        } catch (Exception e) {
            throw new SemanticException("Failed to extract write event info from commit txn message : " + e.getMessage());
        }
    }
    Task<ReplTxnWork> commitTxnTask = TaskFactory.get(work, context.hiveConf);
    // Anyways, if this event gets executed again, it is taken care of.
    if (!context.isDbNameEmpty()) {
        updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, null, null);
    }
    context.log.debug("Added Commit txn task : {}", commitTxnTask.getId());
    if (tasks.isEmpty()) {
        // will be used for setting the last repl id.
        return Collections.singletonList(commitTxnTask);
    }
    DAGTraversal.traverse(tasks, new AddDependencyToLeaves(commitTxnTask));
    return tasks;
}

Example 9 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class TxnHandler method commitTxn.

/**
 * Concurrency/isolation notes:
 * This is mutexed with {@link #openTxns(OpenTxnRequest)} and other {@link #commitTxn(CommitTxnRequest)}
 * operations using select4update on NEXT_TXN_ID. Also, mutexes on TXNS table for specific txnid:X
 * see more notes below.
 * In order to prevent lost updates, we need to determine if any 2 transactions overlap.  Each txn
 * is viewed as an interval [M,N]. M is the txnid and N is taken from the same NEXT_TXN_ID sequence
 * so that we can compare commit time of txn T with start time of txn S.  This sequence can be thought of
 * as a logical time counter. If S.commitTime < T.startTime, T and S do NOT overlap.
 *
 * Motivating example:
 * Suppose we have multi-statement transactions T and S both of which are attempting x = x + 1
 * In order to prevent lost update problem, then the non-overlapping txns must lock in the snapshot
 * that they read appropriately. In particular, if txns do not overlap, then one follows the other
 * (assuming they write the same entity), and thus the 2nd must see changes of the 1st.  We ensure
 * this by locking in snapshot after
 * {@link #openTxns(OpenTxnRequest)} call is made (see org.apache.hadoop.hive.ql.Driver.acquireLocksAndOpenTxn)
 * and mutexing openTxn() with commit(). In other words, once a S.commit() starts we must ensure
 * that txn T which will be considered a later txn, locks in a snapshot that includes the result
 * of S's commit (assuming no other txns).
 * As a counter example, suppose we have S[3,3] and T[4,4] (commitId=txnid means no other transactions
 * were running in parallel). If T and S both locked in the same snapshot (for example commit of
 * txnid:2, which is possible if commitTxn() and openTxnx() is not mutexed)
 * 'x' would be updated to the same value by both, i.e. lost update.
 */
@Override
@RetrySemantics.Idempotent("No-op if already committed")
public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
    char isUpdateDelete = 'N';
    long txnid = rqst.getTxnid();
    long sourceTxnId = -1;
    boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
    boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
    try {
        Connection dbConn = null;
        Statement stmt = null;
        Long commitId = null;
        try {
            lockInternal();
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            if (rqst.isSetReplLastIdInfo()) {
                updateReplId(dbConn, rqst.getReplLastIdInfo());
            }
            if (isReplayedReplTxn) {
                assert (rqst.isSetReplPolicy());
                sourceTxnId = rqst.getTxnid();
                List<Long> targetTxnIds = getTargetTxnIdList(rqst.getReplPolicy(), Collections.singletonList(sourceTxnId), dbConn);
                if (targetTxnIds.isEmpty()) {
                    // Idempotent case where txn was already closed or commit txn event received without
                    // corresponding open txn event.
                    LOG.info("Target txn id is missing for source txn id : " + sourceTxnId + " and repl policy " + rqst.getReplPolicy());
                    return;
                }
                assert targetTxnIds.size() == 1;
                txnid = targetTxnIds.get(0);
            }
            /**
             * Runs at READ_COMMITTED with S4U on TXNS row for "txnid".  S4U ensures that no other
             * operation can change this txn (such acquiring locks). While lock() and commitTxn()
             * should not normally run concurrently (for same txn) but could due to bugs in the client
             * which could then corrupt internal transaction manager state.  Also competes with abortTxn().
             */
            TxnType txnType = getOpenTxnTypeAndLock(stmt, txnid);
            if (txnType == null) {
                // if here, txn was not found (in expected state)
                TxnStatus actualTxnStatus = findTxnState(txnid, stmt);
                if (actualTxnStatus == TxnStatus.COMMITTED) {
                    if (isReplayedReplTxn) {
                        // in case of replication, idempotent is taken care by getTargetTxnId
                        LOG.warn("Invalid state COMMITTED for transactions started using replication replay task");
                    }
                    /**
                     * This makes the operation idempotent
                     * (assume that this is most likely due to retry logic)
                     */
                    LOG.info("Nth commitTxn(" + JavaUtils.txnIdToString(txnid) + ") msg");
                    return;
                }
                raiseTxnUnexpectedState(actualTxnStatus, txnid);
            }
            String conflictSQLSuffix = "FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\"=" + txnid + " AND \"TC_OPERATION_TYPE\" IN (" + OperationType.UPDATE + "," + OperationType.DELETE + ")";
            long tempCommitId = generateTemporaryId();
            if (txnType == TxnType.SOFT_DELETE || txnType == TxnType.COMPACTION) {
                acquireTxnLock(stmt, false);
                commitId = getHighWaterMark(stmt);
            } else if (txnType != TxnType.READ_ONLY && !isReplayedReplTxn) {
                String writeSetInsertSql = "INSERT INTO \"WRITE_SET\" (\"WS_DATABASE\", \"WS_TABLE\", \"WS_PARTITION\"," + "   \"WS_TXNID\", \"WS_COMMIT_ID\", \"WS_OPERATION_TYPE\")" + " SELECT DISTINCT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", \"TC_TXNID\", " + tempCommitId + ", \"TC_OPERATION_TYPE\" ";
                if (isUpdateOrDelete(stmt, conflictSQLSuffix)) {
                    isUpdateDelete = 'Y';
                    // if here it means currently committing txn performed update/delete and we should check WW conflict
                    /**
                     * "select distinct" is used below because
                     * 1. once we get to multi-statement txns, we only care to record that something was updated once
                     * 2. if {@link #addDynamicPartitions(AddDynamicPartitions)} is retried by caller it may create
                     *  duplicate entries in TXN_COMPONENTS
                     * but we want to add a PK on WRITE_SET which won't have unique rows w/o this distinct
                     * even if it includes all of its columns
                     *
                     * First insert into write_set using a temporary commitID, which will be updated in a separate call,
                     * see: {@link #updateWSCommitIdAndCleanUpMetadata(Statement, long, TxnType, Long, long)}}.
                     * This should decrease the scope of the S4U lock on the next_txn_id table.
                     */
                    Savepoint undoWriteSetForCurrentTxn = dbConn.setSavepoint();
                    stmt.executeUpdate(writeSetInsertSql + (useMinHistoryLevel ? conflictSQLSuffix : "FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\"=" + txnid + " AND \"TC_OPERATION_TYPE\" <> " + OperationType.COMPACT));
                    /**
                     * This S4U will mutex with other commitTxn() and openTxns().
                     * -1 below makes txn intervals look like [3,3] [4,4] if all txns are serial
                     * Note: it's possible to have several txns have the same commit id.  Suppose 3 txns start
                     * at the same time and no new txns start until all 3 commit.
                     * We could've incremented the sequence for commitId as well but it doesn't add anything functionally.
                     */
                    acquireTxnLock(stmt, false);
                    commitId = getHighWaterMark(stmt);
                    if (!rqst.isExclWriteEnabled()) {
                        /**
                         * see if there are any overlapping txns that wrote the same element, i.e. have a conflict
                         * Since entire commit operation is mutexed wrt other start/commit ops,
                         * committed.ws_commit_id <= current.ws_commit_id for all txns
                         * thus if committed.ws_commit_id < current.ws_txnid, transactions do NOT overlap
                         * For example, [17,20] is committed, [6,80] is being committed right now - these overlap
                         * [17,20] committed and [21,21] committing now - these do not overlap.
                         * [17,18] committed and [18,19] committing now - these overlap  (here 18 started while 17 was still running)
                         */
                        try (ResultSet rs = checkForWriteConflict(stmt, txnid)) {
                            if (rs.next()) {
                                // found a conflict, so let's abort the txn
                                String committedTxn = "[" + JavaUtils.txnIdToString(rs.getLong(1)) + "," + rs.getLong(2) + "]";
                                StringBuilder resource = new StringBuilder(rs.getString(3)).append("/").append(rs.getString(4));
                                String partitionName = rs.getString(5);
                                if (partitionName != null) {
                                    resource.append('/').append(partitionName);
                                }
                                String msg = "Aborting [" + JavaUtils.txnIdToString(txnid) + "," + commitId + "]" + " due to a write conflict on " + resource + " committed by " + committedTxn + " " + rs.getString(7) + "/" + rs.getString(8);
                                // remove WRITE_SET info for current txn since it's about to abort
                                dbConn.rollback(undoWriteSetForCurrentTxn);
                                LOG.info(msg);
                                // todo: should make abortTxns() write something into TXNS.TXN_META_INFO about this
                                if (abortTxns(dbConn, Collections.singletonList(txnid), false, isReplayedReplTxn) != 1) {
                                    throw new IllegalStateException(msg + " FAILED!");
                                }
                                dbConn.commit();
                                throw new TxnAbortedException(msg);
                            }
                        }
                    }
                } else if (!useMinHistoryLevel) {
                    stmt.executeUpdate(writeSetInsertSql + "FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\"=" + txnid + " AND \"TC_OPERATION_TYPE\" <> " + OperationType.COMPACT);
                    commitId = getHighWaterMark(stmt);
                }
            } else {
                /*
           * current txn didn't update/delete anything (may have inserted), so just proceed with commit
           *
           * We only care about commit id for write txns, so for RO (when supported) txns we don't
           * have to mutex on NEXT_TXN_ID.
           * Consider: if RO txn is after a W txn, then RO's openTxns() will be mutexed with W's
           * commitTxn() because both do S4U on NEXT_TXN_ID and thus RO will see result of W txn.
           * If RO < W, then there is no reads-from relationship.
           * In replication flow we don't expect any write write conflict as it should have been handled at source.
           */
                assert true;
            }
            if (txnType != TxnType.READ_ONLY && !isReplayedReplTxn) {
                moveTxnComponentsToCompleted(stmt, txnid, isUpdateDelete);
            } else if (isReplayedReplTxn) {
                if (rqst.isSetWriteEventInfos()) {
                    String sql = String.format(COMPL_TXN_COMPONENTS_INSERT_QUERY, txnid, quoteChar(isUpdateDelete));
                    try (PreparedStatement pstmt = dbConn.prepareStatement(sql)) {
                        int insertCounter = 0;
                        for (WriteEventInfo writeEventInfo : rqst.getWriteEventInfos()) {
                            pstmt.setString(1, writeEventInfo.getDatabase());
                            pstmt.setString(2, writeEventInfo.getTable());
                            pstmt.setString(3, writeEventInfo.getPartition());
                            pstmt.setLong(4, writeEventInfo.getWriteId());
                            pstmt.addBatch();
                            insertCounter++;
                            if (insertCounter % maxBatchSize == 0) {
                                LOG.debug("Executing a batch of <" + sql + "> queries. Batch size: " + maxBatchSize);
                                pstmt.executeBatch();
                            }
                        }
                        if (insertCounter % maxBatchSize != 0) {
                            LOG.debug("Executing a batch of <" + sql + "> queries. Batch size: " + insertCounter % maxBatchSize);
                            pstmt.executeBatch();
                        }
                    }
                }
                deleteReplTxnMapEntry(dbConn, sourceTxnId, rqst.getReplPolicy());
            }
            updateWSCommitIdAndCleanUpMetadata(stmt, txnid, txnType, commitId, tempCommitId);
            removeTxnsFromMinHistoryLevel(dbConn, ImmutableList.of(txnid));
            if (rqst.isSetKeyValue()) {
                updateKeyValueAssociatedWithTxn(rqst, stmt);
            }
            if (!isHiveReplTxn) {
                createCommitNotificationEvent(dbConn, txnid, txnType);
            }
            LOG.debug("Going to commit");
            dbConn.commit();
            if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_EXT_ON)) {
                Metrics.getOrCreateCounter(MetricsConstants.TOTAL_NUM_COMMITTED_TXNS).inc();
            }
        } catch (SQLException e) {
            LOG.debug("Going to rollback: ", e);
            rollbackDBConn(dbConn);
            checkRetryable(e, "commitTxn(" + rqst + ")");
            throw new MetaException("Unable to update transaction database " + StringUtils.stringifyException(e));
        } finally {
            close(null, stmt, dbConn);
            unlockInternal();
        }
    } catch (RetryException e) {
        commitTxn(rqst);
    }
}

Example 10 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class TestGetAllWriteEventInfo method testGetByTxnIdAndTableName.

@Test
public void testGetByTxnIdAndTableName() throws Exception {
    GetAllWriteEventInfoRequest req = new GetAllWriteEventInfoRequest();
    req.setTxnId(TXN_ID);
    req.setDbName(DB_NAME);
    req.setTableName(TABLE_NAME);
    List<WriteEventInfo> writeEventInfoList = client.getAllWriteEventInfo(req);
    Assert.assertEquals(1, writeEventInfoList.size());
    WriteEventInfo writeEventInfo = writeEventInfoList.get(0);
    Assert.assertEquals(TXN_ID, writeEventInfo.getWriteId());
    Assert.assertEquals(DB_NAME, writeEventInfo.getDatabase());
    Assert.assertEquals(TABLE_NAME, writeEventInfo.getTable());
}