Examples with CompactionInfoStruct org.apache.hadoop.hive.metastore.api.CompactionInfoStruct used on opensource projects

Example 1 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestCommittedCompactionPartition.

@Test
public void testGetLatestCommittedCompactionPartition() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    final String errorMessage = "Dummy error";
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    addFailedCompaction(dbName, tableName, CompactionType.MINOR, partitionName, errorMessage);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    rqst.addToPartitionnames(partitionName);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the first succeeded compaction record", 1, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    final String anotherPartitionName = "ds=yesterday";
    addWaitingForCleaningCompaction(dbName, tableName, CompactionType.MINOR, anotherPartitionName, errorMessage);
    rqst.addToPartitionnames(anotherPartitionName);
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record for each partition", 2, response.getCompactionsSize());
    CompactionInfoStruct lci1 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(partitionName)).findFirst().get();
    assertEquals(1, lci1.getId());
    CompactionInfoStruct lci2 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(anotherPartitionName)).findFirst().get();
    assertEquals(3, lci2.getId());
    // check the result is correct without setting partition names
    rqst.unsetPartitionnames();
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record for each partition", 2, response.getCompactionsSize());
    lci1 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(partitionName)).findFirst().get();
    assertEquals(1, lci1.getId());
    lci2 = response.getCompactions().stream().filter(c -> c.getPartitionname().equals(anotherPartitionName)).findFirst().get();
    assertEquals(3, lci2.getId());
}

Example 2 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestCompactionWithIdFilter.

@Test
public void testGetLatestCompactionWithIdFilter() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    rqst.addToPartitionnames(partitionName);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the second succeeded compaction record", 2, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    // response should only include compaction with id > 2
    rqst.setLastCompactionId(2);
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting no record", 0, response.getCompactionsSize());
}

Example 3 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestSucceededCompaction.

@Test
public void testGetLatestSucceededCompaction() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String partitionName = "ds=today";
    final String errorMessage = "Dummy error";
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    addSucceededCompaction(dbName, tableName, partitionName, CompactionType.MINOR);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    rqst.addToPartitionnames(partitionName);
    txnHandler.getLatestCommittedCompactionInfo(rqst);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the second succeeded compaction record", 2, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    addWaitingForCleaningCompaction(dbName, tableName, CompactionType.MINOR, partitionName, errorMessage);
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single record", 1, response.getCompactionsSize());
    lci = response.getCompactions().get(0);
    assertEquals("Expecting the last compaction record waiting for cleaning", 3, lci.getId());
    assertEquals(partitionName, lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
}

Example 4 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestHiveMetaStoreTxns method testGetLatestCommittedCompactionInfo.

@Test
public void testGetLatestCommittedCompactionInfo() throws Exception {
    final String dbName = "mydb";
    final String tblName = "mytable";
    Database db = new DatabaseBuilder().setName(dbName).build(conf);
    db.unsetCatalogName();
    client.createDatabase(db);
    Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName).addCol("id", "int").addCol("name", "string").setType(TableType.MANAGED_TABLE.name()).build(conf);
    client.createTable(tbl);
    tbl = client.getTable(dbName, tblName);
    client.compact2(tbl.getDbName(), tbl.getTableName(), null, CompactionType.MINOR, new HashMap<>());
    FindNextCompactRequest compactRequest = new FindNextCompactRequest();
    compactRequest.setWorkerId("myworker");
    OptionalCompactionInfoStruct optionalCi = client.findNextCompact(compactRequest);
    client.markCleaned(optionalCi.getCi());
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    // Test invalid inputs
    final String invalidTblName = "invalid";
    rqst.setDbname(dbName);
    Assert.assertThrows(MetaException.class, () -> client.getLatestCommittedCompactionInfo(rqst));
    rqst.setTablename(invalidTblName);
    GetLatestCommittedCompactionInfoResponse response = client.getLatestCommittedCompactionInfo(rqst);
    Assert.assertNotNull(response);
    Assert.assertEquals(0, response.getCompactionsSize());
    // Test normal inputs
    rqst.setTablename(tblName);
    response = client.getLatestCommittedCompactionInfo(rqst);
    Assert.assertNotNull(response);
    Assert.assertEquals(1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    Assert.assertEquals(1, lci.getId());
    Assert.assertNull(lci.getPartitionname());
    Assert.assertEquals(CompactionType.MINOR, lci.getType());
}

Example 5 with CompactionInfoStruct

use of org.apache.hadoop.hive.metastore.api.CompactionInfoStruct in project hive by apache.

the class TestCompactionTxnHandler method testGetLatestCommittedCompaction.

@Test
public void testGetLatestCommittedCompaction() throws Exception {
    final String dbName = "foo";
    final String tableName = "bar";
    final String errorMessage = "Dummy error";
    addSucceededCompaction(dbName, tableName, null, CompactionType.MINOR);
    addFailedCompaction(dbName, tableName, CompactionType.MINOR, null, errorMessage);
    GetLatestCommittedCompactionInfoRequest rqst = new GetLatestCommittedCompactionInfoRequest();
    rqst.setDbname(dbName);
    rqst.setTablename(tableName);
    GetLatestCommittedCompactionInfoResponse response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record", 1, response.getCompactionsSize());
    CompactionInfoStruct lci = response.getCompactions().get(0);
    assertEquals("Expecting the first succeeded compaction record", 1, lci.getId());
    assertNull("Expecting null partitionname for a non-partitioned table", lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
    rqst.setPartitionnames(new ArrayList<>());
    response = txnHandler.getLatestCommittedCompactionInfo(rqst);
    assertNotNull(response);
    assertEquals("Expecting a single compaction record", 1, response.getCompactionsSize());
    lci = response.getCompactions().get(0);
    assertEquals("Expecting the first succeeded compaction record", 1, lci.getId());
    assertEquals(dbName, lci.getDbname());
    assertEquals(tableName, lci.getTablename());
    assertNull("Expecting null partitionname for a non-partitioned table", lci.getPartitionname());
    assertEquals(CompactionType.MINOR, lci.getType());
}