Examples with ValidReadTxnList org.apache.hadoop.hive.common.ValidReadTxnList used on opensource projects

Example 1 with ValidReadTxnList

use of org.apache.hadoop.hive.common.ValidReadTxnList in project hive by apache.

the class TxnHandler method getValidWriteIds.

@Override
@RetrySemantics.ReadOnly
public GetValidWriteIdsResponse getValidWriteIds(GetValidWriteIdsRequest rqst) throws NoSuchTxnException, MetaException {
    try {
        Connection dbConn = null;
        Statement stmt = null;
        ValidTxnList validTxnList;
        // required to get the current state of txns to make validTxnList
        if (rqst.isSetValidTxnList()) {
            validTxnList = new ValidReadTxnList(rqst.getValidTxnList());
        } else {
            // Passing 0 for currentTxn means, this validTxnList is not wrt to any txn
            validTxnList = TxnUtils.createValidReadTxnList(getOpenTxns(), 0);
        }
        try {
            /**
             * This runs at READ_COMMITTED for exactly the same reason as {@link #getOpenTxnsInfo()}
             */
            dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
            stmt = dbConn.createStatement();
            // Get the valid write id list for all the tables read by the current txn
            List<TableValidWriteIds> tblValidWriteIdsList = new ArrayList<>();
            for (String fullTableName : rqst.getFullTableNames()) {
                tblValidWriteIdsList.add(getValidWriteIdsForTable(stmt, fullTableName, validTxnList));
            }
            LOG.debug("Going to rollback");
            dbConn.rollback();
            GetValidWriteIdsResponse owr = new GetValidWriteIdsResponse(tblValidWriteIdsList);
            return owr;
        } catch (SQLException e) {
            LOG.debug("Going to rollback");
            rollbackDBConn(dbConn);
            checkRetryable(dbConn, e, "getValidWriteIds");
            throw new MetaException("Unable to select from transaction database, " + StringUtils.stringifyException(e));
        } finally {
            close(null, stmt, dbConn);
        }
    } catch (RetryException e) {
        return getValidWriteIds(rqst);
    }
}

Example 2 with ValidReadTxnList

use of org.apache.hadoop.hive.common.ValidReadTxnList in project hive by apache.

the class TestAcidUtils method testBestBase.

@Test
public void testBestBase() throws Exception {
    Configuration conf = new Configuration();
    MockFileSystem fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/base_5/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_98_100/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_120_130/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_200/bucket_0", 500, new byte[0]));
    Path part = new MockPath(fs, "/tbl/part1");
    AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_120_130", dir.getCurrentDirectories().get(0).getPath().toString());
    List<FileStatus> obsoletes = dir.getObsolete();
    assertEquals(4, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_98_100", obsoletes.get(3).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("10:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_10", dir.getBaseDirectory().toString());
    assertEquals(0, dir.getCurrentDirectories().size());
    obsoletes = dir.getObsolete();
    assertEquals(1, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(0).getPath().toString());
    assertEquals(0, dir.getOriginalFiles().size());
    /*Single statemnt txns only: since we don't compact a txn range that includes an open txn,
    the existence of delta_120_130 implies that 121 in the exception list is aborted unless
    delta_120_130 is from streaming ingest in which case 121 can be open
    (and thus 122-130 are open too)
    99 here would be Aborted since 121 is minOpenTxn, base_100 is still good
    For multi-statment txns, see HIVE-13369*/
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:121:99:121"));
    assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_120_130", dir.getCurrentDirectories().get(0).getPath().toString());
    obsoletes = dir.getObsolete();
    assertEquals(4, obsoletes.size());
    assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
    assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
    assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
    assertEquals("mock:/tbl/part1/delta_98_100", obsoletes.get(3).getPath().toString());
    boolean gotException = false;
    try {
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("125:5:5"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (125,5).  Oldest available base: " + "mock:/tbl/part1/base_5", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_1_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_12_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    try {
        gotException = false;
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:7:7"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (150,7).  Oldest available base: " + "mock:/tbl/part1/base_25", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, new MockFile("mock:/tbl/part1/delta_2_10/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_25/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    try {
        gotException = false;
        dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:7:7"));
    } catch (IOException e) {
        gotException = true;
        Assert.assertEquals("Not enough history available for (150,7).  Oldest available base: " + "mock:/tbl/part1/base_25", e.getMessage());
    }
    Assert.assertTrue("Expected exception", gotException);
    fs = new MockFileSystem(conf, //non-acid to acid table conversion
    new MockFile("mock:/tbl/part1/base_" + Long.MIN_VALUE + "/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_100/bucket_0", 500, new byte[0]));
    part = new MockPath(fs, "/tbl/part1");
    //note that we don't include current txn of the client in exception list to read-you-writes
    dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("1:" + Long.MAX_VALUE + ":"));
    assertEquals("mock:/tbl/part1/base_" + Long.MIN_VALUE, dir.getBaseDirectory().toString());
    assertEquals(1, dir.getCurrentDirectories().size());
    assertEquals("mock:/tbl/part1/delta_1_1", dir.getCurrentDirectories().get(0).getPath().toString());
    assertEquals(0, dir.getObsolete().size());
}

Example 3 with ValidReadTxnList

use of org.apache.hadoop.hive.common.ValidReadTxnList in project hive by apache.

the class TxnUtils method createValidReadTxnList.

/**
 * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
 * {@link org.apache.hadoop.hive.common.ValidTxnList}.  This assumes that the caller intends to
 * read the files, and thus treats both open and aborted transactions as invalid.
 * @param txns txn list from the metastore
 * @param currentTxn Current transaction that the user has open.  If this is greater than 0 it
 *                   will be removed from the exceptions list so that the user sees his own
 *                   transaction as valid.
 * @return a valid txn list.
 */
public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) {
    /*todo: should highWater be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0
     * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
     * doesn't make sense for Snapshot Isolation.  Of course for Read Committed, the list should
     * inlude the latest committed set.*/
    long highWater = txns.getTxn_high_water_mark();
    List<Long> open = txns.getOpen_txns();
    BitSet abortedBits = BitSet.valueOf(txns.getAbortedBits());
    long[] exceptions = new long[open.size() - (currentTxn > 0 ? 1 : 0)];
    int i = 0;
    for (long txn : open) {
        if (currentTxn > 0 && currentTxn == txn)
            continue;
        exceptions[i++] = txn;
    }
    if (txns.isSetMin_open_txn()) {
        return new ValidReadTxnList(exceptions, abortedBits, highWater, txns.getMin_open_txn());
    } else {
        return new ValidReadTxnList(exceptions, abortedBits, highWater);
    }
}

Example 4 with ValidReadTxnList

use of org.apache.hadoop.hive.common.ValidReadTxnList in project hive by apache.

the class MaterializationsInvalidationCache method addMaterializedView.

/**
 * Adds the materialized view to the cache.
 *
 * @param dbName
 * @param tableName
 * @param tablesUsed tables used by the materialized view
 * @param validTxnList
 * @param opType
 */
private void addMaterializedView(String dbName, String tableName, Set<String> tablesUsed, String validTxnList, OpType opType) {
    if (disable) {
        // Nothing to do
        return;
    }
    // We are going to create the map for each view in the given database
    ConcurrentMap<String, MaterializationInvalidationInfo> cq = new ConcurrentHashMap<String, MaterializationInvalidationInfo>();
    final ConcurrentMap<String, MaterializationInvalidationInfo> prevCq = materializations.putIfAbsent(dbName, cq);
    if (prevCq != null) {
        cq = prevCq;
    }
    // important information in the registry to account for rewriting invalidation
    if (validTxnList == null) {
        // This can happen when the materialized view was created on non-transactional tables
        return;
    }
    if (opType == OpType.CREATE || opType == OpType.ALTER) {
        // You store the materialized view
        cq.put(tableName, new MaterializationInvalidationInfo(tablesUsed, validTxnList));
    } else {
        ValidTxnList txnList = new ValidReadTxnList(validTxnList);
        for (String qNameTableUsed : tablesUsed) {
            // First we insert a new tree set to keep table modifications, unless it already exists
            ConcurrentSkipListMap<Long, Long> modificationsTree = new ConcurrentSkipListMap<Long, Long>();
            final ConcurrentSkipListMap<Long, Long> prevModificationsTree = tableModifications.putIfAbsent(qNameTableUsed, modificationsTree);
            if (prevModificationsTree != null) {
                modificationsTree = prevModificationsTree;
            }
            // check if the MV is still valid.
            try {
                String[] names = qNameTableUsed.split("\\.");
                BasicTxnInfo e = handler.getTxnHandler().getFirstCompletedTransactionForTableAfterCommit(names[0], names[1], txnList);
                if (!e.isIsnull()) {
                    modificationsTree.put(e.getTxnid(), e.getTime());
                    // a modification event that was in the metastore.
                    continue;
                }
            } catch (MetaException ex) {
                LOG.debug("Materialized view " + Warehouse.getQualifiedName(dbName, tableName) + " ignored; error loading view into invalidation cache", ex);
                return;
            }
        }
        // For LOAD, you only add it if it does exist as you might be loading an outdated MV
        cq.putIfAbsent(tableName, new MaterializationInvalidationInfo(tablesUsed, validTxnList));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Cached materialized view for rewriting in invalidation cache: " + Warehouse.getQualifiedName(dbName, tableName));
    }
}

Example 5 with ValidReadTxnList

use of org.apache.hadoop.hive.common.ValidReadTxnList in project hive by apache.

the class MaterializationsInvalidationCache method getInvalidationTime.

private long getInvalidationTime(MaterializationInvalidationInfo materialization) {
    String txnListString = materialization.getValidTxnList();
    if (txnListString == null) {
        // This can happen when the materialization was created on non-transactional tables
        return Long.MIN_VALUE;
    }
    // We will obtain the modification time as follows.
    // First, we obtain the first element after high watermark (if any)
    // Then, we iterate through the elements from min open txn till high
    // watermark, updating the modification time after creation if needed
    ValidTxnList txnList = new ValidReadTxnList(txnListString);
    long firstModificationTimeAfterCreation = 0L;
    for (String qNameTableUsed : materialization.getTablesUsed()) {
        final Entry<Long, Long> tn = tableModifications.get(qNameTableUsed).higherEntry(txnList.getHighWatermark());
        if (tn != null) {
            if (firstModificationTimeAfterCreation == 0L || tn.getValue() < firstModificationTimeAfterCreation) {
                firstModificationTimeAfterCreation = tn.getValue();
            }
        }
        // when this transaction was being executed
        if (txnList.getMinOpenTxn() != null) {
            // Invalid transaction list is sorted
            int pos = 0;
            for (Map.Entry<Long, Long> t : tableModifications.get(qNameTableUsed).subMap(txnList.getMinOpenTxn(), txnList.getHighWatermark()).entrySet()) {
                while (pos < txnList.getInvalidTransactions().length && txnList.getInvalidTransactions()[pos] != t.getKey()) {
                    pos++;
                }
                if (pos >= txnList.getInvalidTransactions().length) {
                    break;
                }
                if (firstModificationTimeAfterCreation == 0L || t.getValue() < firstModificationTimeAfterCreation) {
                    firstModificationTimeAfterCreation = t.getValue();
                }
            }
        }
    }
    return firstModificationTimeAfterCreation;
}