Search in sources :

Example 6 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testProjectionUsingJDO.

@Test
public void testProjectionUsingJDO() throws Throwable {
    // disable direct SQL to make sure
    client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "false");
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Collections.singletonList("sd.location");
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
    // set all the single-valued fields and try using JDO
    request = getGetPartitionsRequest();
    projectSpec = request.getProjectionSpec();
    projectedFields = Arrays.asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.serdeType", "sd.serdeInfo.serializerClass", "sd.serdeInfo.deserializerClass");
    projectSpec.setFieldList(projectedFields);
    response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) PartitionWithoutSD(org.apache.hadoop.hive.metastore.api.PartitionWithoutSD) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) GetPartitionsResponse(org.apache.hadoop.hive.metastore.api.GetPartitionsResponse) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest) PartitionSpec(org.apache.hadoop.hive.metastore.api.PartitionSpec) PartitionSpecWithSharedSD(org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD) Test(org.junit.Test) MetastoreCheckinTest(org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest)

Example 7 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionExcludeParameters.

@Test
public void testPartitionProjectionExcludeParameters() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values"));
    projectSpec.setExcludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%");
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD);
    PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList();
    Assert.assertNull("Partition list composing spec should be null", partitionListComposingSpec);
    for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) {
        Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1"));
        Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2"));
    }
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) PartitionWithoutSD(org.apache.hadoop.hive.metastore.api.PartitionWithoutSD) PartitionListComposingSpec(org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec) GetPartitionsResponse(org.apache.hadoop.hive.metastore.api.GetPartitionsResponse) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest) PartitionSpecWithSharedSD(org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD) Test(org.junit.Test) MetastoreCheckinTest(org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest)

Example 8 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testNestedMultiValuedFieldProjection.

@Test
public void testNestedMultiValuedFieldProjection() throws TException {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("sd.cols.name", "sd.cols.type"));
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols());
    for (FieldSchema col : sharedSD.getCols()) {
        Assert.assertTrue("sd.cols.name was requested but was not returned", col.isSetName());
        Assert.assertTrue("sd.cols.type was requested but was not returned", col.isSetType());
        Assert.assertFalse("sd.cols.comment was not requested but was returned", col.isSetComment());
    }
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) GetPartitionsResponse(org.apache.hadoop.hive.metastore.api.GetPartitionsResponse) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest) PartitionSpecWithSharedSD(org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD) Test(org.junit.Test) MetastoreCheckinTest(org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest)

Example 9 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method runWithInvalidFieldNames.

private void runWithInvalidFieldNames(List<String> values) throws TException {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(values);
    client.getPartitionsWithSpecs(request);
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest)

Example 10 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method getPartitionsWithVals.

private void getPartitionsWithVals(List<String> filters, int expectedPartitions) throws TException {
    // get partitions from "trusted" API
    List<Partition> partitions = client.listPartitions(dbName, tblName, filters, (short) -1);
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("sd.location"));
    request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_VALUES);
    request.getFilterSpec().setFilters(filters);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertNotNull(response);
    if (expectedPartitions > 0) {
        PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
        Assert.assertNotNull(partitionSpecWithSharedSD);
        Assert.assertEquals("Invalid number of partitions returned", expectedPartitions, partitionSpecWithSharedSD.getPartitionsSize());
        verifyLocations(partitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions());
    } else {
        Assert.assertTrue("Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty());
    }
}
Also used : GetProjectionsSpec(org.apache.hadoop.hive.metastore.api.GetProjectionsSpec) Partition(org.apache.hadoop.hive.metastore.api.Partition) GetPartitionsResponse(org.apache.hadoop.hive.metastore.api.GetPartitionsResponse) GetPartitionsRequest(org.apache.hadoop.hive.metastore.api.GetPartitionsRequest) PartitionSpecWithSharedSD(org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD)

Aggregations

GetPartitionsRequest (org.apache.hadoop.hive.metastore.api.GetPartitionsRequest)16 GetProjectionsSpec (org.apache.hadoop.hive.metastore.api.GetProjectionsSpec)16 GetPartitionsResponse (org.apache.hadoop.hive.metastore.api.GetPartitionsResponse)13 PartitionSpecWithSharedSD (org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD)13 Test (org.junit.Test)10 MetastoreCheckinTest (org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest)9 PartitionWithoutSD (org.apache.hadoop.hive.metastore.api.PartitionWithoutSD)8 StorageDescriptor (org.apache.hadoop.hive.metastore.api.StorageDescriptor)7 Partition (org.apache.hadoop.hive.metastore.api.Partition)5 PartitionSpec (org.apache.hadoop.hive.metastore.api.PartitionSpec)4 PartitionListComposingSpec (org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec)3 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)2 GetPartitionsFilterSpec (org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec)2 IOException (java.io.IOException)1 SQLException (java.sql.SQLException)1 ConfigValSecurityException (org.apache.hadoop.hive.metastore.api.ConfigValSecurityException)1 InvalidObjectException (org.apache.hadoop.hive.metastore.api.InvalidObjectException)1 InvalidOperationException (org.apache.hadoop.hive.metastore.api.InvalidOperationException)1 MetaException (org.apache.hadoop.hive.metastore.api.MetaException)1 NoSuchObjectException (org.apache.hadoop.hive.metastore.api.NoSuchObjectException)1