Examples with GetProjectionsSpec org.apache.hadoop.hive.metastore.api.GetProjectionsSpec used on opensource projects

Example 1 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionAllSingleValuedFields.

@Test
public void testPartitionProjectionAllSingleValuedFields() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Arrays.asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib");
    // TODO directSQL does not support serdeType, serializerClass and deserializerClass in serdeInfo
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
}

Example 2 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method getPartitionsWithExpr.

private void getPartitionsWithExpr(List<String> filters, int expectedPartition) throws TException {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("sd.location"));
    request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_EXPR);
    request.getFilterSpec().setFilters(filters);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertNotNull(response);
    if (expectedPartition > 0) {
        PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
        Assert.assertNotNull(partitionSpecWithSharedSD);
        Assert.assertEquals("Invalid number of partitions returned", expectedPartition, partitionSpecWithSharedSD.getPartitionsSize());
    } else {
        Assert.assertTrue("Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty());
    }
}

Example 3 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionIncludeParameters.

@Test
public void testPartitionProjectionIncludeParameters() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values"));
    projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%");
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD);
    PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList();
    Assert.assertNull("Partition list composing spec should be null since all the " + "partitions are expected to be in sharedSD spec", partitionListComposingSpec);
    for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) {
        Assert.assertTrue("included parameter key is not found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1"));
        Assert.assertTrue("included parameter key is not found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2"));
        Assert.assertEquals("Additional parameters returned other than inclusion keys", 2, retPartion.getParameters().size());
    }
}

Example 4 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionAllMultiValuedFields.

@Test
public void testPartitionProjectionAllMultiValuedFields() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Arrays.asList("values", "parameters", "sd.cols", "sd.bucketCols", "sd.sortCols", "sd.parameters", "sd.skewedInfo", "sd.serdeInfo.parameters");
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    Assert.assertEquals(origPartitions.size(), partitionSpecWithSharedSD.getPartitions().size());
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    for (int i = 0; i < origPartitions.size(); i++) {
        Partition origPartition = origPartitions.get(i);
        PartitionWithoutSD retPartition = partitionSpecWithSharedSD.getPartitions().get(i);
        for (String projectedField : projectedFields) {
            switch(projectedField) {
                case "values":
                    validateList(origPartition.getValues(), retPartition.getValues());
                    break;
                case "parameters":
                    validateMap(origPartition.getParameters(), retPartition.getParameters());
                    break;
                case "sd.cols":
                    validateList(origPartition.getSd().getCols(), sharedSD.getCols());
                    break;
                case "sd.bucketCols":
                    validateList(origPartition.getSd().getBucketCols(), sharedSD.getBucketCols());
                    break;
                case "sd.sortCols":
                    validateList(origPartition.getSd().getSortCols(), sharedSD.getSortCols());
                    break;
                case "sd.parameters":
                    validateMap(origPartition.getSd().getParameters(), sharedSD.getParameters());
                    break;
                case "sd.skewedInfo":
                    if (!origPartition.getSd().getSkewedInfo().getSkewedColNames().isEmpty()) {
                        validateList(origPartition.getSd().getSkewedInfo().getSkewedColNames(), sharedSD.getSkewedInfo().getSkewedColNames());
                    }
                    if (!origPartition.getSd().getSkewedInfo().getSkewedColValues().isEmpty()) {
                        for (int i1 = 0; i1 < origPartition.getSd().getSkewedInfo().getSkewedColValuesSize(); i1++) {
                            validateList(origPartition.getSd().getSkewedInfo().getSkewedColValues().get(i1), sharedSD.getSkewedInfo().getSkewedColValues().get(i1));
                        }
                    }
                    if (!origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps().isEmpty()) {
                        validateMap(origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps(), sharedSD.getSkewedInfo().getSkewedColValueLocationMaps());
                    }
                    break;
                case "sd.serdeInfo.parameters":
                    validateMap(origPartition.getSd().getSerdeInfo().getParameters(), sharedSD.getSerdeInfo().getParameters());
                    break;
                default:
                    throw new IllegalArgumentException("Invalid field " + projectedField);
            }
        }
    }
}

Example 5 with GetProjectionsSpec

use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method getGetPartitionsRequest.

private GetPartitionsRequest getGetPartitionsRequest() {
    GetPartitionsRequest request = new GetPartitionsRequest();
    request.setProjectionSpec(new GetProjectionsSpec());
    request.setFilterSpec(new GetPartitionsFilterSpec());
    request.setTblName(tblName);
    request.setDbName(dbName);
    return request;
}