use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.
the class TestGetPartitionsUsingProjectionAndFilterSpecs method getPartitionsWithNames.
private void getPartitionsWithNames(List<String> names, int expectedPartitionCount) throws TException {
GetPartitionsRequest request = getGetPartitionsRequest();
GetProjectionsSpec projectSpec = request.getProjectionSpec();
projectSpec.setFieldList(Arrays.asList("sd.location"));
request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_NAMES);
request.getFilterSpec().setFilters(names);
GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
Assert.assertNotNull(response);
if (expectedPartitionCount > 0) {
PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
Assert.assertNotNull(partitionSpecWithSharedSD);
Assert.assertEquals("Invalid number of partitions returned", expectedPartitionCount, partitionSpecWithSharedSD.getPartitionsSize());
List<Partition> origPartitions = client.getPartitionsByNames(dbName, tblName, names);
verifyLocations(origPartitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions());
} else {
Assert.assertTrue("Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty());
}
}
use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.
the class TestHiveMetaStore method testGetPartitionsWithSpec.
@Test
public void testGetPartitionsWithSpec() throws Throwable {
// create a table with multiple partitions
List<Partition> createdPartitions = setupProjectionTestTable();
Table tbl = client.getTable("compdb", "comptbl");
GetPartitionsRequest request = new GetPartitionsRequest();
GetProjectionsSpec projectSpec = new GetProjectionsSpec();
projectSpec.setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "lastAccessTime", "sd.location", "values", "createTime", "sd.serdeInfo.serializationLib", "sd.cols"));
projectSpec.setExcludeParamKeyPattern("exclude%");
GetPartitionsFilterSpec filter = new GetPartitionsFilterSpec();
request.setDbName("compdb");
request.setTblName("comptbl");
request.setFilterSpec(filter);
request.setProjectionSpec(projectSpec);
GetPartitionsResponse response;
try {
response = client.getPartitionsWithSpecs(request);
} catch (Exception ex) {
ex.printStackTrace();
LOG.error("Exception while retriveing partitions", ex);
throw ex;
}
Assert.assertEquals(1, response.getPartitionSpecSize());
PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
Assert.assertNotNull(partitionSpecWithSharedSD.getSd());
StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
Assert.assertEquals("Root location should be set to table location", tbl.getSd().getLocation(), sharedSD.getLocation());
Assert.assertFalse("Fields which are not requested should not be set", sharedSD.isSetParameters());
Assert.assertNotNull("serializationLib class was requested but was not found in the returned partition", sharedSD.getSerdeInfo().getSerializationLib());
Assert.assertNotNull("db name was requested but was not found in the returned partition", response.getPartitionSpec().get(0).getDbName());
Assert.assertNotNull("Table name was requested but was not found in the returned partition", response.getPartitionSpec().get(0).getTableName());
Assert.assertTrue("sd.cols was requested but was not found in the returned response", partitionSpecWithSharedSD.getSd().isSetCols());
List<FieldSchema> origSdCols = createdPartitions.get(0).getSd().getCols();
Assert.assertEquals("Size of the requested sd.cols should be same", origSdCols.size(), partitionSpecWithSharedSD.getSd().getCols().size());
for (int i = 0; i < origSdCols.size(); i++) {
FieldSchema origFs = origSdCols.get(i);
FieldSchema returnedFs = partitionSpecWithSharedSD.getSd().getCols().get(i);
Assert.assertEquals("Field schemas returned different than expected", origFs, returnedFs);
}
/*Assert
.assertNotNull("Catalog name was requested but was not found in the returned partition",
response.getPartitionSpec().get(0).getCatName());*/
List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
Assert.assertEquals(createdPartitions.size(), partitionWithoutSDS.size());
for (int i = 0; i < createdPartitions.size(); i++) {
Partition origPartition = createdPartitions.get(i);
PartitionWithoutSD returnedPartitionWithoutSD = partitionWithoutSDS.get(i);
Assert.assertEquals(String.format("Location returned for Partition %d is not correct", i), origPartition.getSd().getLocation(), sharedSD.getLocation() + returnedPartitionWithoutSD.getRelativePath());
Assert.assertTrue("createTime was request but is not set", returnedPartitionWithoutSD.isSetCreateTime());
Assert.assertTrue("Partition parameters were requested but are not set", returnedPartitionWithoutSD.isSetParameters());
// first partition has parameters set
if (i == 0) {
Assert.assertTrue("partition parameters not set", returnedPartitionWithoutSD.getParameters().containsKey("key1"));
Assert.assertEquals("partition parameters does not contain included keys", "val1", returnedPartitionWithoutSD.getParameters().get("key1"));
// excluded parameter should not be returned
Assert.assertFalse("Excluded parameter key returned", returnedPartitionWithoutSD.getParameters().containsKey("excludeKey1"));
Assert.assertFalse("Excluded parameter key returned", returnedPartitionWithoutSD.getParameters().containsKey("excludeKey2"));
}
List<String> returnedVals = returnedPartitionWithoutSD.getValues();
List<String> actualVals = origPartition.getValues();
for (int j = 0; j < actualVals.size(); j++) {
Assert.assertEquals(actualVals.get(j), returnedVals.get(j));
}
}
}
use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.
the class TestGetPartitionsUsingProjectionAndFilterSpecs method testParameterExpansion.
@Test
public void testParameterExpansion() throws TException {
GetPartitionsRequest request = getGetPartitionsRequest();
GetProjectionsSpec projectSpec = request.getProjectionSpec();
projectSpec.setFieldList(Arrays.asList("sd.cols", "sd.serdeInfo"));
GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols());
Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", origPartitions.get(0).getSd().getCols(), sharedSD.getCols());
Assert.assertNotNull("sd.serdeInfo were requested but was not returned", sharedSD.getSerdeInfo());
Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", origPartitions.get(0).getSd().getSerdeInfo(), sharedSD.getSerdeInfo());
}
use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.
the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionEmptySpec.
@Test
public void testPartitionProjectionEmptySpec() throws Throwable {
GetPartitionsRequest request = getGetPartitionsRequest();
GetProjectionsSpec projectSpec = request.getProjectionSpec();
projectSpec.setFieldList(new ArrayList<>(0));
projectSpec.setExcludeParamKeyPattern("exclude%");
GetPartitionsResponse response;
response = client.getPartitionsWithSpecs(request);
Assert.assertEquals(1, response.getPartitionSpec().size());
PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
Assert.assertNotNull(sharedSD);
// everything except location in sharedSD should be same
StorageDescriptor origSd = origPartitions.get(0).getSd().deepCopy();
origSd.unsetLocation();
StorageDescriptor sharedSDCopy = sharedSD.deepCopy();
sharedSDCopy.unsetLocation();
Assert.assertEquals(origSd, sharedSDCopy);
List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
Assert.assertNotNull(partitionWithoutSDS);
Assert.assertEquals("Unexpected number of partitions returned", origPartitions.size(), partitionWithoutSDS.size());
for (int i = 0; i < origPartitions.size(); i++) {
Partition origPartition = origPartitions.get(i);
PartitionWithoutSD retPartition = partitionWithoutSDS.get(i);
Assert.assertEquals(origPartition.getCreateTime(), retPartition.getCreateTime());
Assert.assertEquals(origPartition.getLastAccessTime(), retPartition.getLastAccessTime());
Assert.assertEquals(origPartition.getSd().getLocation(), sharedSD.getLocation() + retPartition.getRelativePath());
validateMap(origPartition.getParameters(), retPartition.getParameters());
validateList(origPartition.getValues(), retPartition.getValues());
}
}
use of org.apache.hadoop.hive.metastore.api.GetProjectionsSpec in project hive by apache.
the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionIncludeExcludeParameters.
@Test
public void testPartitionProjectionIncludeExcludeParameters() throws Throwable {
GetPartitionsRequest request = getGetPartitionsRequest();
GetProjectionsSpec projectSpec = request.getProjectionSpec();
projectSpec.setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values"));
// test parameter key inclusion using setIncludeParamKeyPattern
projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%");
projectSpec.setExcludeParamKeyPattern("%key1%");
GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD);
PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList();
Assert.assertNull("Partition list composing spec should be null since all the " + "partitions are expected to be in sharedSD spec", partitionListComposingSpec);
for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) {
Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1"));
Assert.assertTrue("included parameter key is not found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2"));
Assert.assertEquals("Additional parameters returned other than inclusion keys", 1, retPartion.getParameters().size());
}
}
Aggregations