Search in sources :

Example 1 with GetPartitionsFilterSpec

use of org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method getGetPartitionsRequest.

private GetPartitionsRequest getGetPartitionsRequest() {
    GetPartitionsRequest request = new GetPartitionsRequest();
    request.setProjectionSpec(new GetProjectionsSpec());
    request.setFilterSpec(new GetPartitionsFilterSpec());
    request.setTblName(tblName);
    request.setDbName(dbName);
    return request;
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest) GetPartitionsFilterSpec(org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec)

Example 2 with GetPartitionsFilterSpec

use of org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec in project hive by apache.

the class TestHiveMetaStore method testGetPartitionsWithSpec.

@Test
public void testGetPartitionsWithSpec() throws Throwable {
    // create a table with multiple partitions
    List<Partition> createdPartitions = setupProjectionTestTable();
    Table tbl = client.getTable("compdb", "comptbl");
    GetPartitionsRequest request = new GetPartitionsRequest();
    GetProjectionsSpec projectSpec = new GetProjectionsSpec();
    projectSpec.setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "lastAccessTime", "sd.location", "values", "createTime", "sd.serdeInfo.serializationLib", "sd.cols"));
    projectSpec.setExcludeParamKeyPattern("exclude%");
    GetPartitionsFilterSpec filter = new GetPartitionsFilterSpec();
    request.setDbName("compdb");
    request.setTblName("comptbl");
    request.setFilterSpec(filter);
    request.setProjectionSpec(projectSpec);
    GetPartitionsResponse response;
    try {
        response = client.getPartitionsWithSpecs(request);
    } catch (Exception ex) {
        ex.printStackTrace();
        LOG.error("Exception while retriveing partitions", ex);
        throw ex;
    }
    Assert.assertEquals(1, response.getPartitionSpecSize());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    Assert.assertNotNull(partitionSpecWithSharedSD.getSd());
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertEquals("Root location should be set to table location", tbl.getSd().getLocation(), sharedSD.getLocation());
    Assert.assertFalse("Fields which are not requested should not be set", sharedSD.isSetParameters());
    Assert.assertNotNull("serializationLib class was requested but was not found in the returned partition", sharedSD.getSerdeInfo().getSerializationLib());
    Assert.assertNotNull("db name was requested but was not found in the returned partition", response.getPartitionSpec().get(0).getDbName());
    Assert.assertNotNull("Table name was requested but was not found in the returned partition", response.getPartitionSpec().get(0).getTableName());
    Assert.assertTrue("sd.cols was requested but was not found in the returned response", partitionSpecWithSharedSD.getSd().isSetCols());
    List<FieldSchema> origSdCols = createdPartitions.get(0).getSd().getCols();
    Assert.assertEquals("Size of the requested sd.cols should be same", origSdCols.size(), partitionSpecWithSharedSD.getSd().getCols().size());
    for (int i = 0; i < origSdCols.size(); i++) {
        FieldSchema origFs = origSdCols.get(i);
        FieldSchema returnedFs = partitionSpecWithSharedSD.getSd().getCols().get(i);
        Assert.assertEquals("Field schemas returned different than expected", origFs, returnedFs);
    }
    /*Assert
        .assertNotNull("Catalog name was requested but was not found in the returned partition",
            response.getPartitionSpec().get(0).getCatName());*/
    List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertEquals(createdPartitions.size(), partitionWithoutSDS.size());
    for (int i = 0; i < createdPartitions.size(); i++) {
        Partition origPartition = createdPartitions.get(i);
        PartitionWithoutSD returnedPartitionWithoutSD = partitionWithoutSDS.get(i);
        Assert.assertEquals(String.format("Location returned for Partition %d is not correct", i), origPartition.getSd().getLocation(), sharedSD.getLocation() + returnedPartitionWithoutSD.getRelativePath());
        Assert.assertTrue("createTime was request but is not set", returnedPartitionWithoutSD.isSetCreateTime());
        Assert.assertTrue("Partition parameters were requested but are not set", returnedPartitionWithoutSD.isSetParameters());
        // first partition has parameters set
        if (i == 0) {
            Assert.assertTrue("partition parameters not set", returnedPartitionWithoutSD.getParameters().containsKey("key1"));
            Assert.assertEquals("partition parameters does not contain included keys", "val1", returnedPartitionWithoutSD.getParameters().get("key1"));
            // excluded parameter should not be returned
            Assert.assertFalse("Excluded parameter key returned", returnedPartitionWithoutSD.getParameters().containsKey("excludeKey1"));
            Assert.assertFalse("Excluded parameter key returned", returnedPartitionWithoutSD.getParameters().containsKey("excludeKey2"));
        }
        List<String> returnedVals = returnedPartitionWithoutSD.getValues();
        List<String> actualVals = origPartition.getValues();
        for (int j = 0; j < actualVals.size(); j++) {
            Assert.assertEquals(actualVals.get(j), returnedVals.get(j));
        }
    }
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) Partition(org.apache.hadoop.hive.metastore.api.Partition) SourceTable(org.apache.hadoop.hive.metastore.api.SourceTable) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) GetPartitionsResponse(org.apache.hadoop.hive.metastore.api.GetPartitionsResponse) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest) PartitionSpecWithSharedSD(org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) InvalidOperationException(org.apache.hadoop.hive.metastore.api.InvalidOperationException) ConfigValSecurityException(org.apache.hadoop.hive.metastore.api.ConfigValSecurityException) SQLException(java.sql.SQLException) UnknownDBException(org.apache.hadoop.hive.metastore.api.UnknownDBException) TException(org.apache.thrift.TException) IOException(java.io.IOException) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) GetPartitionsFilterSpec(org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec) PartitionWithoutSD(org.apache.hadoop.hive.metastore.api.PartitionWithoutSD) Test(org.junit.Test)

Aggregations

GetPartitionsFilterSpec (org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec)2 GetPartitionsRequest (org.apache.hadoop.hive.metastore.api.GetPartitionsRequest)2 GetProjectionsSpec (org.apache.hadoop.hive.metastore.api.GetProjectionsSpec)2 IOException (java.io.IOException)1 SQLException (java.sql.SQLException)1 ConfigValSecurityException (org.apache.hadoop.hive.metastore.api.ConfigValSecurityException)1 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)1 GetPartitionsResponse (org.apache.hadoop.hive.metastore.api.GetPartitionsResponse)1 InvalidObjectException (org.apache.hadoop.hive.metastore.api.InvalidObjectException)1 InvalidOperationException (org.apache.hadoop.hive.metastore.api.InvalidOperationException)1 MetaException (org.apache.hadoop.hive.metastore.api.MetaException)1 NoSuchObjectException (org.apache.hadoop.hive.metastore.api.NoSuchObjectException)1 Partition (org.apache.hadoop.hive.metastore.api.Partition)1 PartitionSpecWithSharedSD (org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD)1 PartitionWithoutSD (org.apache.hadoop.hive.metastore.api.PartitionWithoutSD)1 SourceTable (org.apache.hadoop.hive.metastore.api.SourceTable)1 StorageDescriptor (org.apache.hadoop.hive.metastore.api.StorageDescriptor)1 Table (org.apache.hadoop.hive.metastore.api.Table)1 UnknownDBException (org.apache.hadoop.hive.metastore.api.UnknownDBException)1 TException (org.apache.thrift.TException)1