use of org.apache.hive.streaming.HiveStreamingConnection in project hive by apache.
the class TestCompactor method testCleanAbortCompactAfterAbortTwoPartitions.
@Test
public void testCleanAbortCompactAfterAbortTwoPartitions() throws Exception {
String dbName = "default";
String tblName = "cws";
HiveStreamingConnection connection1 = prepareTableTwoPartitionsAndConnection(dbName, tblName, 1);
HiveStreamingConnection connection2 = prepareTableTwoPartitionsAndConnection(dbName, tblName, 1);
// to skip optimization introduced in HIVE-22122
executeStatementOnDriver("truncate " + tblName, driver);
connection1.beginTransaction();
connection1.write("1,1".getBytes());
connection1.write("2,2".getBytes());
connection1.abortTransaction();
connection2.beginTransaction();
connection2.write("1,3".getBytes());
connection2.write("2,3".getBytes());
connection2.write("3,3".getBytes());
connection2.abortTransaction();
assertAndCompactCleanAbort(dbName, tblName, false, false);
connection1.close();
connection2.close();
}
use of org.apache.hive.streaming.HiveStreamingConnection in project hive by apache.
the class TestCompactor method testCleanAbortCompactAfterAbort.
@Test
public void testCleanAbortCompactAfterAbort() throws Exception {
String dbName = "default";
String tblName = "cws";
// Create three folders with two different transactions
HiveStreamingConnection connection1 = prepareTableAndConnection(dbName, tblName, 1);
HiveStreamingConnection connection2 = prepareTableAndConnection(dbName, tblName, 1);
// to skip optimization introduced in HIVE-22122
executeStatementOnDriver("truncate " + tblName, driver);
connection1.beginTransaction();
connection1.write("1,1".getBytes());
connection1.write("2,2".getBytes());
connection1.abortTransaction();
connection2.beginTransaction();
connection2.write("1,3".getBytes());
connection2.write("2,3".getBytes());
connection2.write("3,3".getBytes());
connection2.abortTransaction();
assertAndCompactCleanAbort(dbName, tblName, false, false);
connection1.close();
connection2.close();
}
use of org.apache.hive.streaming.HiveStreamingConnection in project hive by apache.
the class TestCompactor method testCleanAbortCorrectlyCleaned.
@Test
public void testCleanAbortCorrectlyCleaned() throws Exception {
// Test that at commit the tables are cleaned properly
String dbName = "default";
String tblName = "cws";
HiveStreamingConnection connection = prepareTableAndConnection(dbName, tblName, 1);
connection.beginTransaction();
connection.write("1,1".getBytes());
connection.write("2,2".getBytes());
int count = TestTxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_OPERATION_TYPE='i'");
// We should have 1 row corresponding to the aborted transaction
Assert.assertEquals(TestTxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1, count);
connection.commitTransaction();
// After commit the row should have been deleted
count = TestTxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_OPERATION_TYPE='i'");
Assert.assertEquals(TestTxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 0, count);
}
use of org.apache.hive.streaming.HiveStreamingConnection in project hive by apache.
the class TestCompactor method testCleanAbortCompactAfter1stCommitAbort.
@Test
public void testCleanAbortCompactAfter1stCommitAbort() throws Exception {
String dbName = "default";
String tblName = "cws";
HiveStreamingConnection connection = prepareTableAndConnection(dbName, tblName, 2);
connection.beginTransaction();
connection.write("1,1".getBytes());
connection.write("2,2".getBytes());
connection.abortTransaction();
connection.beginTransaction();
connection.write("3,2".getBytes());
connection.write("3,3".getBytes());
connection.commitTransaction();
assertAndCompactCleanAbort(dbName, tblName, true, true);
connection.close();
}
use of org.apache.hive.streaming.HiveStreamingConnection in project hive by apache.
the class TestCompactor method testHeartbeatShutdownOnFailedCompaction.
@Test
public void testHeartbeatShutdownOnFailedCompaction() throws Exception {
String dbName = "default";
String tblName = "compaction_test";
executeStatementOnDriver("drop table if exists " + tblName, driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " + " PARTITIONED BY(bkt INT)" + // currently ACID requires table to be bucketed
" CLUSTERED BY(a) INTO 4 BUCKETS" + " STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver);
StrictDelimitedInputWriter writer = StrictDelimitedInputWriter.newBuilder().withFieldDelimiter(',').build();
HiveStreamingConnection connection = HiveStreamingConnection.newBuilder().withDatabase(dbName).withTable(tblName).withStaticPartitionValues(Arrays.asList("0")).withAgentInfo("UT_" + Thread.currentThread().getName()).withHiveConf(conf).withRecordWriter(writer).connect();
connection.beginTransaction();
connection.write("55, 'London'".getBytes());
connection.commitTransaction();
connection.beginTransaction();
connection.write("56, 'Paris'".getBytes());
connection.commitTransaction();
connection.close();
executeStatementOnDriver("INSERT INTO TABLE " + tblName + " PARTITION(bkt=1)" + " values(57, 'Budapest')", driver);
executeStatementOnDriver("INSERT INTO TABLE " + tblName + " PARTITION(bkt=1)" + " values(58, 'Milano')", driver);
execSelectAndDumpData("select * from " + tblName, driver, "Dumping data for " + tblName + " after load:");
TxnStore txnHandler = TxnUtils.getTxnStore(conf);
// Commit will throw an exception
IMetaStoreClient mockedClient = Mockito.spy(new HiveMetaStoreClient(conf));
doThrow(new RuntimeException("Simulating RuntimeException from CompactionTxn.commit")).when(mockedClient).commitTxn(Mockito.anyLong());
doAnswer(invocation -> {
Object o = invocation.callRealMethod();
// Check if the heartbeating is running
Assert.assertTrue(Thread.getAllStackTraces().keySet().stream().anyMatch(k -> k.getName().contains("CompactionTxn Heartbeater")));
return o;
}).when(mockedClient).openTxn(any(), any());
// Do a major compaction
CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR);
rqst.setPartitionname("bkt=0");
txnHandler.compact(rqst);
Worker worker = Mockito.spy(new Worker());
worker.setThreadId((int) worker.getId());
worker.setConf(conf);
worker.init(new AtomicBoolean(true));
FieldSetter.setField(worker, RemoteCompactorThread.class.getDeclaredField("msc"), mockedClient);
worker.run();
// Check if the transaction was opened
verify(mockedClient, times(1)).openTxn(any(), any());
// Check if the heartbeating is properly terminated
Assert.assertTrue(Thread.getAllStackTraces().keySet().stream().noneMatch(k -> k.getName().contains("CompactionTxn Heartbeater")));
}
Aggregations