Examples with BasicTxnInfo org.apache.hadoop.hive.metastore.api.BasicTxnInfo used on opensource projects

Example 1 with BasicTxnInfo

use of org.apache.hadoop.hive.metastore.api.BasicTxnInfo in project hive by apache.

the class TestMetaStoreMaterializationsCacheCleaner method testCleanerScenario1.

@Test
public void testCleanerScenario1() throws Exception {
    // create mock raw store
    Configuration conf = new Configuration();
    conf.set("metastore.materializations.invalidation.impl", "DISABLE");
    // create mock handler
    final IHMSHandler handler = mock(IHMSHandler.class);
    // initialize invalidation cache (set conf to disable)
    MaterializationsInvalidationCache.get().init(conf, handler);
    // This is a dummy test, invalidation cache is not supposed to
    // record any information.
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 1, 1);
    int id = 2;
    BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, id, id);
    // Create tbl2 (nothing to do)
    id = 3;
    BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, id, id);
    // Cleanup (current = 4, duration = 4) -> Does nothing
    long removed = MaterializationsInvalidationCache.get().cleanup(0L);
    Assert.assertEquals(0L, removed);
    // Create mv1
    Table mv1 = mock(Table.class);
    when(mv1.getDbName()).thenReturn(DB_NAME);
    when(mv1.getTableName()).thenReturn(MV_NAME_1);
    CreationMetadata mockCM1 = new CreationMetadata(DB_NAME, MV_NAME_1, ImmutableSet.of(DB_NAME + "." + TBL_NAME_1, DB_NAME + "." + TBL_NAME_2));
    // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE)
    mockCM1.setValidTxnList("3:" + Long.MAX_VALUE + "::");
    when(mv1.getCreationMetadata()).thenReturn(mockCM1);
    MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), mockCM1.getTablesUsed(), mockCM1.getValidTxnList());
    Map<String, Materialization> invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1));
    Assert.assertTrue(invalidationInfos.isEmpty());
    id = 10;
    BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, id, id);
    id = 9;
    BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, id, id);
    // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3
    removed = MaterializationsInvalidationCache.get().cleanup(8L);
    Assert.assertEquals(0L, removed);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1));
    Assert.assertTrue(invalidationInfos.isEmpty());
    // Create mv2
    Table mv2 = mock(Table.class);
    when(mv2.getDbName()).thenReturn(DB_NAME);
    when(mv2.getTableName()).thenReturn(MV_NAME_2);
    CreationMetadata mockCM2 = new CreationMetadata(DB_NAME, MV_NAME_2, ImmutableSet.of(DB_NAME + "." + TBL_NAME_1, DB_NAME + "." + TBL_NAME_2));
    // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE)
    mockCM2.setValidTxnList("10:" + Long.MAX_VALUE + "::");
    when(mv2.getCreationMetadata()).thenReturn(mockCM2);
    MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), mockCM2.getTablesUsed(), mockCM2.getValidTxnList());
    when(mv2.getCreationMetadata()).thenReturn(mockCM2);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertTrue(invalidationInfos.isEmpty());
    // Create tbl3 (nothing to do)
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_3, 11, 11);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_3, 18, 18);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 14, 14);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 17, 17);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, 16, 16);
    // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11
    removed = MaterializationsInvalidationCache.get().cleanup(16L);
    Assert.assertEquals(0L, removed);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertTrue(invalidationInfos.isEmpty());
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 12, 12);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, 15, 15);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, 7, 7);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertTrue(invalidationInfos.isEmpty());
    // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18
    removed = MaterializationsInvalidationCache.get().cleanup(20L);
    Assert.assertEquals(0L, removed);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertTrue(invalidationInfos.isEmpty());
    // Cleanup (current = 28, duration = 4) -> Removes txn9
    removed = MaterializationsInvalidationCache.get().cleanup(24L);
    Assert.assertEquals(0L, removed);
}

Example 2 with BasicTxnInfo

use of org.apache.hadoop.hive.metastore.api.BasicTxnInfo in project hive by apache.

the class TestMetaStoreMaterializationsCacheCleaner method testCleanerScenario2.

@Test
public void testCleanerScenario2() throws Exception {
    // create mock raw store
    Configuration conf = new Configuration();
    conf.set("metastore.materializations.invalidation.impl", "DEFAULT");
    // create mock handler
    final IHMSHandler handler = mock(IHMSHandler.class);
    // initialize invalidation cache (set conf to default)
    MaterializationsInvalidationCache.get().init(conf, handler);
    // Scenario consists of the following steps:
    // Create tbl1
    // (t = 1) Insert row in tbl1
    // (t = 2) Insert row in tbl1
    // Create tbl2
    // (t = 3) Insert row in tbl2
    // Cleanup (current = 4, duration = 4) -> Does nothing
    // Create mv1
    // (t = 10) Insert row in tbl2
    // (t = 9) Insert row in tbl1 (out of order)
    // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3
    // Create mv2
    // Create tbl3
    // (t = 11) Insert row in tbl3
    // (t = 18) Insert row in tbl3
    // (t = 14) Insert row in tbl1
    // (t = 17) Insert row in tbl1
    // (t = 16) Insert row in tbl2
    // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11
    // (t = 12) Insert row in tbl1
    // (t = 15) Insert row in tbl2
    // (t = 7) Insert row in tbl2
    // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18
    // Create tbl1 (nothing to do)
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 1, 1);
    int id = 2;
    BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, id, id);
    // Create tbl2 (nothing to do)
    id = 3;
    BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, id, id);
    // Cleanup (current = 4, duration = 4) -> Does nothing
    long removed = MaterializationsInvalidationCache.get().cleanup(0L);
    Assert.assertEquals(0L, removed);
    // Create mv1
    Table mv1 = mock(Table.class);
    when(mv1.getDbName()).thenReturn(DB_NAME);
    when(mv1.getTableName()).thenReturn(MV_NAME_1);
    CreationMetadata mockCM1 = new CreationMetadata(DB_NAME, MV_NAME_1, ImmutableSet.of(DB_NAME + "." + TBL_NAME_1, DB_NAME + "." + TBL_NAME_2));
    // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE)
    mockCM1.setValidTxnList("3:" + Long.MAX_VALUE + "::");
    when(mv1.getCreationMetadata()).thenReturn(mockCM1);
    MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), mockCM1.getTablesUsed(), mockCM1.getValidTxnList());
    Map<String, Materialization> invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1));
    Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
    id = 10;
    BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, id, id);
    id = 9;
    BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, id, id);
    // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3
    removed = MaterializationsInvalidationCache.get().cleanup(8L);
    Assert.assertEquals(3L, removed);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1));
    Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
    // Create mv2
    Table mv2 = mock(Table.class);
    when(mv2.getDbName()).thenReturn(DB_NAME);
    when(mv2.getTableName()).thenReturn(MV_NAME_2);
    CreationMetadata mockCM2 = new CreationMetadata(DB_NAME, MV_NAME_2, ImmutableSet.of(DB_NAME + "." + TBL_NAME_1, DB_NAME + "." + TBL_NAME_2));
    // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE)
    mockCM2.setValidTxnList("10:" + Long.MAX_VALUE + "::");
    when(mv2.getCreationMetadata()).thenReturn(mockCM2);
    MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), mockCM2.getTablesUsed(), mockCM2.getValidTxnList());
    when(mv2.getCreationMetadata()).thenReturn(mockCM2);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
    Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
    // Create tbl3 (nothing to do)
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_3, 11, 11);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_3, 18, 18);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 14, 14);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 17, 17);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, 16, 16);
    // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11
    removed = MaterializationsInvalidationCache.get().cleanup(16L);
    Assert.assertEquals(2L, removed);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
    Assert.assertEquals(14L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_1, 12, 12);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, 15, 15);
    MaterializationsInvalidationCache.get().notifyTableModification(DB_NAME, TBL_NAME_2, 7, 7);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
    Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
    // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18
    removed = MaterializationsInvalidationCache.get().cleanup(20L);
    Assert.assertEquals(6L, removed);
    invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
    Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
    Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
    // Cleanup (current = 28, duration = 4) -> Removes txn9
    removed = MaterializationsInvalidationCache.get().cleanup(24L);
    Assert.assertEquals(0L, removed);
}

Example 3 with BasicTxnInfo

use of org.apache.hadoop.hive.metastore.api.BasicTxnInfo in project hive by apache.

the class TxnHandler method getFirstCompletedTransactionForTableAfterCommit.

/**
 * Gets the information of the first transaction for the given table
 * after the transaction with the input id was committed (if any).
 */
@Override
@RetrySemantics.ReadOnly
public BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit(String inputDbName, String inputTableName, ValidTxnList txnList) throws MetaException {
    final List<Long> openTxns = Arrays.asList(ArrayUtils.toObject(txnList.getInvalidTransactions()));
    Connection dbConn = null;
    Statement stmt = null;
    ResultSet rs = null;
    try {
        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
        stmt = dbConn.createStatement();
        stmt.setMaxRows(1);
        String s = "select ctc_timestamp, ctc_txnid, ctc_database, ctc_table " + "from COMPLETED_TXN_COMPONENTS " + "where ctc_database=" + quoteString(inputDbName) + " and ctc_table=" + quoteString(inputTableName) + " and ctc_txnid > " + txnList.getHighWatermark() + (txnList.getInvalidTransactions().length == 0 ? " " : " or ctc_txnid IN(" + StringUtils.join(",", openTxns) + ") ") + "order by ctc_timestamp asc";
        if (LOG.isDebugEnabled()) {
            LOG.debug("Going to execute query <" + s + ">");
        }
        rs = stmt.executeQuery(s);
        if (!rs.next()) {
            return new BasicTxnInfo(true);
        }
        final BasicTxnInfo txnInfo = new BasicTxnInfo(false);
        txnInfo.setTime(rs.getTimestamp(1, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime());
        txnInfo.setTxnid(rs.getLong(2));
        txnInfo.setDbname(rs.getString(3));
        txnInfo.setTablename(rs.getString(4));
        return txnInfo;
    } catch (SQLException ex) {
        LOG.warn("getLastCompletedTransactionForTable failed due to " + getMessage(ex), ex);
        throw new MetaException("Unable to retrieve commits information due to " + StringUtils.stringifyException(ex));
    } finally {
        close(rs, stmt, dbConn);
    }
}

Example 4 with BasicTxnInfo

use of org.apache.hadoop.hive.metastore.api.BasicTxnInfo in project hive by apache.

the class MaterializationsInvalidationCache method addMaterializedView.

/**
 * Adds the materialized view to the cache.
 *
 * @param dbName
 * @param tableName
 * @param tablesUsed tables used by the materialized view
 * @param validTxnList
 * @param opType
 */
private void addMaterializedView(String dbName, String tableName, Set<String> tablesUsed, String validTxnList, OpType opType) {
    if (disable) {
        // Nothing to do
        return;
    }
    // We are going to create the map for each view in the given database
    ConcurrentMap<String, MaterializationInvalidationInfo> cq = new ConcurrentHashMap<String, MaterializationInvalidationInfo>();
    final ConcurrentMap<String, MaterializationInvalidationInfo> prevCq = materializations.putIfAbsent(dbName, cq);
    if (prevCq != null) {
        cq = prevCq;
    }
    // important information in the registry to account for rewriting invalidation
    if (validTxnList == null) {
        // This can happen when the materialized view was created on non-transactional tables
        return;
    }
    if (opType == OpType.CREATE || opType == OpType.ALTER) {
        // You store the materialized view
        cq.put(tableName, new MaterializationInvalidationInfo(tablesUsed, validTxnList));
    } else {
        ValidTxnList txnList = new ValidReadTxnList(validTxnList);
        for (String qNameTableUsed : tablesUsed) {
            // First we insert a new tree set to keep table modifications, unless it already exists
            ConcurrentSkipListMap<Long, Long> modificationsTree = new ConcurrentSkipListMap<Long, Long>();
            final ConcurrentSkipListMap<Long, Long> prevModificationsTree = tableModifications.putIfAbsent(qNameTableUsed, modificationsTree);
            if (prevModificationsTree != null) {
                modificationsTree = prevModificationsTree;
            }
            // check if the MV is still valid.
            try {
                String[] names = qNameTableUsed.split("\\.");
                BasicTxnInfo e = handler.getTxnHandler().getFirstCompletedTransactionForTableAfterCommit(names[0], names[1], txnList);
                if (!e.isIsnull()) {
                    modificationsTree.put(e.getTxnid(), e.getTime());
                    // a modification event that was in the metastore.
                    continue;
                }
            } catch (MetaException ex) {
                LOG.debug("Materialized view " + Warehouse.getQualifiedName(dbName, tableName) + " ignored; error loading view into invalidation cache", ex);
                return;
            }
        }
        // For LOAD, you only add it if it does exist as you might be loading an outdated MV
        cq.putIfAbsent(tableName, new MaterializationInvalidationInfo(tablesUsed, validTxnList));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Cached materialized view for rewriting in invalidation cache: " + Warehouse.getQualifiedName(dbName, tableName));
    }
}

Example 5 with BasicTxnInfo

use of org.apache.hadoop.hive.metastore.api.BasicTxnInfo in project hive by apache.

the class TestMetaStoreMaterializationsCacheCleaner method createTxnInfo.

private static BasicTxnInfo createTxnInfo(String dbName, String tableName, int i) {
    BasicTxnInfo r = new BasicTxnInfo();
    r.setDbname(dbName);
    r.setTablename(tableName);
    r.setTxnid(i);
    r.setTime(i);
    return r;
}