Search in sources :

Example 1 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestCommittedCompactionPartition.

@Test
public void testGetLatestCommittedCompactionPartition() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    final String errorMessage = "Dummy error";
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    addFailedCompaction(dbName, tableName, CompactionType.MINOR, partitionName, errorMessage);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    rqst.addToPartitionnames(partitionName);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the first succeeded compaction record", 1, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    final String anotherPartitionName = "ds=yesterday";
    addWaitingForCleaningCompaction(dbName, tableName, CompactionType.MINOR, anotherPartitionName, errorMessage);
    rqst.addToPartitionnames(anotherPartitionName);
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record for each partition", 2, response.getCompactionsSize());
    CompactionInfoStruct lci1 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(partitionName)).findFirst().get();
    assertEquals(1, lci1.getId());
    CompactionInfoStruct lci2 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(anotherPartitionName)).findFirst().get();
    assertEquals(3, lci2.getId());
    // check the result is correct without setting partition names
    rqst.unsetPartitionnames();
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record for each partition", 2, response.getCompactionsSize());
    lci1 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(partitionName)).findFirst().get();
    assertEquals(1, lci1.getId());
    lci2 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(anotherPartitionName)).findFirst().get();
    assertEquals(3, lci2.getId());
}
Also used : LockType(org.apache.hadoop.hive.metastore.api.LockType) CompactionRequest(org.apache.hadoop.hive.metastore.api.CompactionRequest) Arrays(java.util.Arrays) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) TestTxnDbUtil(org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil) SortedSet(java.util.SortedSet) FindNextCompactRequest(org.apache.hadoop.hive.metastore.api.FindNextCompactRequest) OpenTxnsResponse(org.apache.hadoop.hive.metastore.api.OpenTxnsResponse) GetLatestCommittedCompactionInfoResponse(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse) LockRequest(org.apache.hadoop.hive.metastore.api.LockRequest) CompactionType(org.apache.hadoop.hive.metastore.api.CompactionType) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) AllocateTableWriteIdsRequest(org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest) CommitTxnRequest(org.apache.hadoop.hive.metastore.api.CommitTxnRequest) DataOperationType(org.apache.hadoop.hive.metastore.api.DataOperationType) ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) LockLevel(org.apache.hadoop.hive.metastore.api.LockLevel) COMPACTOR_INITIATOR_FAILED_THRESHOLD(org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD) LockState(org.apache.hadoop.hive.metastore.api.LockState) GetOpenTxnsResponse(org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse) After(org.junit.After) LockComponent(org.apache.hadoop.hive.metastore.api.LockComponent) Assert.fail(org.junit.Assert.fail) ShowCompactResponse(org.apache.hadoop.hive.metastore.api.ShowCompactResponse) Before(org.junit.Before) AbortTxnRequest(org.apache.hadoop.hive.metastore.api.AbortTxnRequest) OpenTxnRequest(org.apache.hadoop.hive.metastore.api.OpenTxnRequest) CompactionInfoStruct(org.apache.hadoop.hive.metastore.api.CompactionInfoStruct) Assert.assertNotNull(org.junit.Assert.assertNotNull) LockResponse(org.apache.hadoop.hive.metastore.api.LockResponse) HiveConf(org.apache.hadoop.hive.conf.HiveConf) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Collectors(java.util.stream.Collectors) AllocateTableWriteIdsResponse(org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse) COMPACTOR_INITIATOR_FAILED_RETRY_TIME(org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_RETRY_TIME) TimeUnit(java.util.concurrent.TimeUnit) GetLatestCommittedCompactionInfoRequest(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest) List(java.util.List) MetastoreConf(org.apache.hadoop.hive.metastore.conf.MetastoreConf) Assert.assertNull(org.junit.Assert.assertNull) Assert.assertFalse(org.junit.Assert.assertFalse) AddDynamicPartitions(org.apache.hadoop.hive.metastore.api.AddDynamicPartitions) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) GetLatestCommittedCompactionInfoRequest(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest) CompactionInfoStruct(org.apache.hadoop.hive.metastore.api.CompactionInfoStruct) GetLatestCommittedCompactionInfoResponse(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse) Test(org.junit.Test)

Example 2 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestCompactionWithIdFilter.

@Test
public void testGetLatestCompactionWithIdFilter() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    rqst.addToPartitionnames(partitionName);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the second succeeded compaction record", 2, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    // response should only include compaction with id > 2
    rqst.setLastCompactionId(2);
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting no record", 0, response.getCompactionsSize());
}
Also used : GetLatestCommittedCompactionInfoRequest(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest) CompactionInfoStruct(org.apache.hadoop.hive.metastore.api.CompactionInfoStruct) GetLatestCommittedCompactionInfoResponse(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse) Test(org.junit.Test)

Example 3 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestSucceededCompaction.

@Test
public void testGetLatestSucceededCompaction() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    final String errorMessage = "Dummy error";
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    rqst.addToPartitionnames(partitionName);
    txnHandler.getLatestCommittedCompactionInfo(rqst);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the second succeeded compaction record", 2, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    addWaitingForCleaningCompaction(dbName, tableName, CompactionType.MINOR, partitionName, errorMessage);
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single record", 1, response.getCompactionsSize());
    lci = response.getCompactions().get(0);
    assertEquals("Expecting the last compaction record waiting for cleaning", 3, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
}
Also used : GetLatestCommittedCompactionInfoRequest(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest) CompactionInfoStruct(org.apache.hadoop.hive.metastore.api.CompactionInfoStruct) GetLatestCommittedCompactionInfoResponse(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse) Test(org.junit.Test)

Example 4 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestHiveMetaStoreTxns method testGetLatestCommittedCompactionInfo.

@Test
public void testGetLatestCommittedCompactionInfo() throws Exception {
    final String dbName = "mydb";
    final String tblName = "mytable";
    Database db = new DatabaseBuilder().setName(dbName).build(conf);
    db.unsetCatalogName();
    client.createDatabase(db);
    Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName).addCol("id", "int").addCol("name", "string").setType(TableType.MANAGED_TABLE.name()).build(conf);
    client.createTable(tbl);
    tbl = client.getTable(dbName, tblName);
    client.compact2(tbl.getDbName(), tbl.getTableName(), null, CompactionType.MINOR, new HashMap<>());
    FindNextCompactRequest compactRequest = new FindNextCompactRequest();
    compactRequest.setWorkerId("myworker");
    OptionalCompactionInfoStruct optionalCi = client.findNextCompact(compactRequest);
    client.markCleaned(optionalCi.getCi());
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    // Test invalid inputs
    final String invalidTblName = "invalid";
    rqst.setDbname(dbName);
    Assert.assertThrows(MetaException.class, () -> client.getLatestCommittedCompactionInfo(rqst));
    rqst.setTablename(invalidTblName);
    GetLatestCommittedCompactionInfoResponse response = client.getLatestCommittedCompactionInfo(rqst);
    Assert.assertNotNull(response);
    Assert.assertEquals(0, response.getCompactionsSize());
    // Test normal inputs
    rqst.setTablename(tblName);
    response = client.getLatestCommittedCompactionInfo(rqst);
    Assert.assertNotNull(response);
    Assert.assertEquals(1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    Assert.assertEquals(1, lci.getId());
    Assert.assertNull(lci.getPartitionname());
    Assert.assertEquals(CompactionType.MINOR, lci.getType());
}
Also used : DatabaseBuilder(org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder) Table(org.apache.hadoop.hive.metastore.api.Table) GetLatestCommittedCompactionInfoRequest(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest) Database(org.apache.hadoop.hive.metastore.api.Database) CompactionInfoStruct(org.apache.hadoop.hive.metastore.api.CompactionInfoStruct) OptionalCompactionInfoStruct(org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct) FindNextCompactRequest(org.apache.hadoop.hive.metastore.api.FindNextCompactRequest) OptionalCompactionInfoStruct(org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct) TableBuilder(org.apache.hadoop.hive.metastore.client.builder.TableBuilder) GetLatestCommittedCompactionInfoResponse(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse) MetastoreUnitTest(org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest) Test(org.junit.Test)

Example 5 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestCommittedCompaction.

@Test
public void testGetLatestCommittedCompaction() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String errorMessage = "Dummy error";
    addSucceededCompaction(dbName, tableName, null, CompactionType.MINOR);
    addFailedCompaction(dbName, tableName, CompactionType.MINOR, null, errorMessage);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the first succeeded compaction record", 1, lci.getId());
    assertNull("Expecting null partitionname for a non-partitioned table", lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    rqst.setPartitionnames(new ArrayList<>());
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record", 1, response.getCompactionsSize());
    lci = response.getCompactions().get(0);
    assertEquals("Expecting the first succeeded compaction record", 1, lci.getId());
    assertEquals(dbName, lci.getDbname());
    assertEquals(tableName, lci.getTablename());
    assertNull("Expecting null partitionname for a non-partitioned table", lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
}
Also used : GetLatestCommittedCompactionInfoRequest(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest) CompactionInfoStruct(org.apache.hadoop.hive.metastore.api.CompactionInfoStruct) GetLatestCommittedCompactionInfoResponse(org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse) Test(org.junit.Test)

Aggregations

CompactionInfoStruct (org.apache.hadoop.hive.metastore.api.CompactionInfoStruct)8 GetLatestCommittedCompactionInfoResponse (org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse)6 GetLatestCommittedCompactionInfoRequest (org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest)5 Test (org.junit.Test)5 OptionalCompactionInfoStruct (org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct)3 ArrayList (java.util.ArrayList)2 FindNextCompactRequest (org.apache.hadoop.hive.metastore.api.FindNextCompactRequest)2 Connection (java.sql.Connection)1 PreparedStatement (java.sql.PreparedStatement)1 ResultSet (java.sql.ResultSet)1 SQLException (java.sql.SQLException)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Set (java.util.Set)1 SortedSet (java.util.SortedSet)1 TreeSet (java.util.TreeSet)1 TimeUnit (java.util.concurrent.TimeUnit)1 Collectors (java.util.stream.Collectors)1