Examples with ShowLocksResponseElement org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement used on opensource projects

Example 11 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method testDynamicPartitionInsert.

/**
   * Check that DP with partial spec properly updates TXN_COMPONENTS
   * @throws Exception
   */
@Test
public void testDynamicPartitionInsert() throws Exception {
    dropTable(new String[] { "target" });
    checkCmdOnDriver(driver.run("create table target (a int, b int) " + "partitioned by (p int, q int) clustered by (a) into 2  buckets " + "stored as orc TBLPROPERTIES ('transactional'='true')"));
    long txnid1 = txnMgr.openTxn(ctx, "T1");
    checkCmdOnDriver(driver.compileAndRespond("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
    List<ShowLocksResponseElement> locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    //table is empty, so can only lock the table
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks);
    Assert.assertEquals("HIVE_LOCKS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + TxnDbUtil.queryToString("select * from HIVE_LOCKS"), 1, TxnDbUtil.countQueryAgent("select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1));
    txnMgr.rollbackTxn();
    Assert.assertEquals("TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), 0, TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
    //now actually write to table to generate some partitions
    checkCmdOnDriver(driver.run("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)"));
    driver.run("select count(*) from target");
    List<String> r = new ArrayList<>();
    driver.getResults(r);
    Assert.assertEquals("", "4", r.get(0));
    //look in COMPLETED_TXN_COMPONENTS because driver.run() committed!!!!
    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1 + 1) + "): " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), //2 distinct partitions created
    2, //txnid+1 because we want txn used by previous driver.run("insert....)
    TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1)));
    long txnid2 = txnMgr.openTxn(ctx, "T1");
    checkCmdOnDriver(driver.compileAndRespond("insert into target partition(p=1,q) values (10,2,2), (30,4,2), (50,6,3), (70,8,2)"));
    txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
    locks = getLocks(txnMgr);
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    //Plan is using DummyPartition, so can only lock the table... unfortunately
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks);
    AddDynamicPartitions adp = new AddDynamicPartitions(txnid2, "default", "target", Arrays.asList("p=1/q=2", "p=1/q=2"));
    adp.setOperationType(DataOperationType.INSERT);
    txnHandler.addDynamicPartitions(adp);
    Assert.assertEquals("TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), //2 distinct partitions modified
    2, TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
    txnMgr.commitTxn();
}

Example 12 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager2 method insertOverwritePartitionedCreate.

@Test
public void insertOverwritePartitionedCreate() throws Exception {
    dropTable(new String[] { "T4" });
    CommandProcessorResponse cpr = driver.run("create table if not exists T4 (name string, gpa double) partitioned by (age int)");
    checkCmdOnDriver(cpr);
    cpr = driver.run("create table if not exists T5(name string, age int, gpa double)");
    checkCmdOnDriver(cpr);
    cpr = driver.compileAndRespond("INSERT OVERWRITE TABLE T4 PARTITION (age) SELECT name, age, gpa FROM T5");
    checkCmdOnDriver(cpr);
    txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 2, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T5", null, locks);
    checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T4", null, locks);
    txnMgr.getLockManager().releaseLocks(ctx.getHiveLocks());
    Assert.assertEquals("Lock remained", 0, getLocks().size());
    cpr = driver.run("drop table if exists T5");
    checkCmdOnDriver(cpr);
    cpr = driver.run("drop table if exists T4");
    checkCmdOnDriver(cpr);
}

Example 13 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestDbTxnManager method testWriteDynamicPartition.

@Test
public void testWriteDynamicPartition() throws Exception {
    WriteEntity we = addDynamicPartitionedOutput(newTable(true), WriteEntity.WriteType.INSERT);
    QueryPlan qp = new MockQueryPlan(this);
    txnMgr.openTxn(ctx, "fred");
    txnMgr.acquireLocks(qp, ctx, "fred");
    List<HiveLock> locks = ctx.getHiveLocks();
    Assert.assertEquals(1, locks.size());
    /*Assert.assertEquals(1,
        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
    */
    // Make sure we're locking the whole table, since this is dynamic partitioning
    ShowLocksResponse rsp = ((DbLockManager) txnMgr.getLockManager()).getLocks();
    List<ShowLocksResponseElement> elms = rsp.getLocks();
    Assert.assertEquals(1, elms.size());
    Assert.assertNotNull(elms.get(0).getTablename());
    Assert.assertNull(elms.get(0).getPartname());
    txnMgr.commitTxn();
    locks = txnMgr.getLockManager().getLocks(false, false);
    Assert.assertEquals(0, locks.size());
}

Example 14 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestStreaming method testHeartbeat.

@Test
public void testHeartbeat() throws Exception {
    HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, dbName2, tblName2, null);
    StreamingConnection connection = endPt.newConnection(false, "UT_" + Thread.currentThread().getName());
    DelimitedInputWriter writer = new DelimitedInputWriter(fieldNames2, ",", endPt, connection);
    TransactionBatch txnBatch = connection.fetchTransactionBatch(5, writer);
    txnBatch.beginNextTransaction();
    //todo: this should ideally check Transaction heartbeat as well, but heartbeat
    //timestamp is not reported yet
    //GetOpenTxnsInfoResponse txnresp = msClient.showTxns();
    ShowLocksRequest request = new ShowLocksRequest();
    request.setDbname(dbName2);
    request.setTablename(tblName2);
    ShowLocksResponse response = msClient.showLocks(request);
    Assert.assertEquals("Wrong nubmer of locks: " + response, 1, response.getLocks().size());
    ShowLocksResponseElement lock = response.getLocks().get(0);
    long acquiredAt = lock.getAcquiredat();
    long heartbeatAt = lock.getLastheartbeat();
    txnBatch.heartbeat();
    response = msClient.showLocks(request);
    Assert.assertEquals("Wrong number of locks2: " + response, 1, response.getLocks().size());
    lock = response.getLocks().get(0);
    Assert.assertEquals("Acquired timestamp didn't match", acquiredAt, lock.getAcquiredat());
    Assert.assertTrue("Expected new heartbeat (" + lock.getLastheartbeat() + ") == old heartbeat(" + heartbeatAt + ")", lock.getLastheartbeat() == heartbeatAt);
    txnBatch.close();
    int txnBatchSize = 200;
    txnBatch = connection.fetchTransactionBatch(txnBatchSize, writer);
    for (int i = 0; i < txnBatchSize; i++) {
        txnBatch.beginNextTransaction();
        if (i % 47 == 0) {
            txnBatch.heartbeat();
        }
        if (i % 10 == 0) {
            txnBatch.abort();
        } else {
            txnBatch.commit();
        }
        if (i % 37 == 0) {
            txnBatch.heartbeat();
        }
    }
}

Example 15 with ShowLocksResponseElement

use of org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement in project hive by apache.

the class TestTxnHandler method showLocks.

@Test
public void showLocks() throws Exception {
    long begining = System.currentTimeMillis();
    LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
    comp.setOperationType(DataOperationType.NO_TXN);
    List<LockComponent> components = new ArrayList<LockComponent>(1);
    components.add(comp);
    LockRequest req = new LockRequest(components, "me", "localhost");
    LockResponse res = txnHandler.lock(req);
    // Open txn
    long txnid = openTxn();
    comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "mydb");
    comp.setTablename("mytable");
    comp.setOperationType(DataOperationType.SELECT);
    components = new ArrayList<LockComponent>(1);
    components.add(comp);
    req = new LockRequest(components, "me", "localhost");
    req.setTxnid(txnid);
    res = txnHandler.lock(req);
    // Locks not associated with a txn
    components = new ArrayList<LockComponent>(1);
    comp = new LockComponent(LockType.SHARED_READ, LockLevel.PARTITION, "yourdb");
    comp.setTablename("yourtable");
    comp.setPartitionname("yourpartition");
    comp.setOperationType(DataOperationType.INSERT);
    components.add(comp);
    req = new LockRequest(components, "you", "remotehost");
    res = txnHandler.lock(req);
    ShowLocksResponse rsp = txnHandler.showLocks(new ShowLocksRequest());
    List<ShowLocksResponseElement> locks = rsp.getLocks();
    assertEquals(3, locks.size());
    boolean[] saw = new boolean[locks.size()];
    for (int i = 0; i < saw.length; i++) saw[i] = false;
    for (ShowLocksResponseElement lock : locks) {
        if (lock.getLockid() == 1) {
            assertEquals(0, lock.getTxnid());
            assertEquals("mydb", lock.getDbname());
            assertNull(lock.getTablename());
            assertNull(lock.getPartname());
            assertEquals(LockState.ACQUIRED, lock.getState());
            assertEquals(LockType.EXCLUSIVE, lock.getType());
            assertTrue(lock.toString(), 0 != lock.getLastheartbeat());
            assertTrue("Expected acquired at " + lock.getAcquiredat() + " to be between " + begining + " and " + System.currentTimeMillis(), begining <= lock.getAcquiredat() && System.currentTimeMillis() >= lock.getAcquiredat());
            assertEquals("me", lock.getUser());
            assertEquals("localhost", lock.getHostname());
            saw[0] = true;
        } else if (lock.getLockid() == 2) {
            assertEquals(1, lock.getTxnid());
            assertEquals("mydb", lock.getDbname());
            assertEquals("mytable", lock.getTablename());
            assertNull(lock.getPartname());
            assertEquals(LockState.WAITING, lock.getState());
            assertEquals(LockType.SHARED_READ, lock.getType());
            assertTrue(lock.toString(), 0 == lock.getLastheartbeat() && lock.getTxnid() != 0);
            assertEquals(0, lock.getAcquiredat());
            assertEquals("me", lock.getUser());
            assertEquals("localhost", lock.getHostname());
            saw[1] = true;
        } else if (lock.getLockid() == 3) {
            assertEquals(0, lock.getTxnid());
            assertEquals("yourdb", lock.getDbname());
            assertEquals("yourtable", lock.getTablename());
            assertEquals("yourpartition", lock.getPartname());
            assertEquals(LockState.ACQUIRED, lock.getState());
            assertEquals(LockType.SHARED_READ, lock.getType());
            assertTrue(lock.toString(), begining <= lock.getLastheartbeat() && System.currentTimeMillis() >= lock.getLastheartbeat());
            assertTrue(begining <= lock.getAcquiredat() && System.currentTimeMillis() >= lock.getAcquiredat());
            assertEquals("you", lock.getUser());
            assertEquals("remotehost", lock.getHostname());
            saw[2] = true;
        } else {
            fail("Unknown lock id");
        }
    }
    for (int i = 0; i < saw.length; i++) assertTrue("Didn't see lock id " + i, saw[i]);
}