Examples with GetPartitionsResponse org.apache.hadoop.hive.metastore.api.GetPartitionsResponse used on opensource projects

Example 6 with GetPartitionsResponse

use of org.apache.hadoop.hive.metastore.api.GetPartitionsResponse in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testProjectionUsingJDO.

@Test
public void testProjectionUsingJDO() throws Throwable {
    // disable direct SQL to make sure
    client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "false");
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Collections.singletonList("sd.location");
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
    // set all the single-valued fields and try using JDO
    request = getGetPartitionsRequest();
    projectSpec = request.getProjectionSpec();
    projectedFields = Arrays.asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.serdeType", "sd.serdeInfo.serializerClass", "sd.serdeInfo.deserializerClass");
    projectSpec.setFieldList(projectedFields);
    response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
}

Example 7 with GetPartitionsResponse

use of org.apache.hadoop.hive.metastore.api.GetPartitionsResponse in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionExcludeParameters.

@Test
public void testPartitionProjectionExcludeParameters() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values"));
    projectSpec.setExcludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%");
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD);
    PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList();
    Assert.assertNull("Partition list composing spec should be null", partitionListComposingSpec);
    for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) {
        Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1"));
        Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2"));
    }
}

Example 8 with GetPartitionsResponse

use of org.apache.hadoop.hive.metastore.api.GetPartitionsResponse in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testNestedMultiValuedFieldProjection.

@Test
public void testNestedMultiValuedFieldProjection() throws TException {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("sd.cols.name", "sd.cols.type"));
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols());
    for (FieldSchema col : sharedSD.getCols()) {
        Assert.assertTrue("sd.cols.name was requested but was not returned", col.isSetName());
        Assert.assertTrue("sd.cols.type was requested but was not returned", col.isSetType());
        Assert.assertFalse("sd.cols.comment was not requested but was returned", col.isSetComment());
    }
}

Example 9 with GetPartitionsResponse

use of org.apache.hadoop.hive.metastore.api.GetPartitionsResponse in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method getPartitionsWithVals.

private void getPartitionsWithVals(List<String> filters, int expectedPartitions) throws TException {
    // get partitions from "trusted" API
    List<Partition> partitions = client.listPartitions(dbName, tblName, filters, (short) -1);
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    projectSpec.setFieldList(Arrays.asList("sd.location"));
    request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_VALUES);
    request.getFilterSpec().setFilters(filters);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertNotNull(response);
    if (expectedPartitions > 0) {
        PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
        Assert.assertNotNull(partitionSpecWithSharedSD);
        Assert.assertEquals("Invalid number of partitions returned", expectedPartitions, partitionSpecWithSharedSD.getPartitionsSize());
        verifyLocations(partitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions());
    } else {
        Assert.assertTrue("Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty());
    }
}

Example 10 with GetPartitionsResponse

use of org.apache.hadoop.hive.metastore.api.GetPartitionsResponse in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testNonStandardPartitions.

@Test
public void testNonStandardPartitions() throws TException {
    String testTblName = "test_non_standard";
    new TableBuilder().setTableName(testTblName).setDbName(dbName).addCol("ns_c1", "string", "comment 1").addCol("ns_c2", "int", "comment 2").addPartCol("part", "string").addPartCol("city", "string").addBucketCol("ns_c1").addSortCol("ns_c2", 1).addTableParam("tblparamKey", "Partitions of this table are not located within table directory").create(client, conf);
    Table table = client.getTable(dbName, testTblName);
    Assert.assertNotNull("Unable to create a test table ", table);
    List<Partition> partitions = new ArrayList<>();
    partitions.add(createPartition(Arrays.asList("p1", "SanFrancisco"), table));
    partitions.add(createPartition(Arrays.asList("p1", "PaloAlto"), table));
    partitions.add(createPartition(Arrays.asList("p2", "Seattle"), table));
    partitions.add(createPartition(Arrays.asList("p2", "Phoenix"), table));
    client.add_partitions(partitions);
    // change locations of two of the partitions outside table directory
    List<Partition> testPartitions = client.listPartitions(dbName, testTblName, (short) -1);
    Assert.assertEquals(4, testPartitions.size());
    Partition p1 = testPartitions.get(2);
    p1.getSd().setLocation("/tmp/some_other_location/part=p2/city=Seattle");
    Partition p2 = testPartitions.get(3);
    p2.getSd().setLocation("/tmp/some_other_location/part=p2/city=Phoenix");
    client.alter_partitions(dbName, testTblName, Arrays.asList(p1, p2));
    GetPartitionsRequest request = getGetPartitionsRequest();
    request.getProjectionSpec().setFieldList(Arrays.asList("values", "sd"));
    request.setDbName(dbName);
    request.setTblName(testTblName);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertNotNull("Response should have returned partition specs", response.getPartitionSpec());
    Assert.assertEquals("We should have two partition specs", 2, response.getPartitionSpec().size());
    Assert.assertNotNull("One SharedSD spec is expected", response.getPartitionSpec().get(0).getSharedSDPartitionSpec());
    Assert.assertNotNull("One composing spec is expected", response.getPartitionSpec().get(1).getPartitionList());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec();
    Assert.assertNotNull("sd was requested but not returned", partitionSpecWithSharedSD.getSd());
    Assert.assertEquals("shared SD should have table location", table.getSd().getLocation(), partitionSpecWithSharedSD.getSd().getLocation());
    List<List<String>> expectedVals = new ArrayList<>(2);
    expectedVals.add(Arrays.asList("p1", "PaloAlto"));
    expectedVals.add(Arrays.asList("p1", "SanFrancisco"));
    for (int i = 0; i < partitionSpecWithSharedSD.getPartitions().size(); i++) {
        PartitionWithoutSD retPartition = partitionSpecWithSharedSD.getPartitions().get(i);
        Assert.assertEquals(2, retPartition.getValuesSize());
        validateList(expectedVals.get(i), retPartition.getValues());
        Assert.assertNull("parameters were not requested so should have been null", retPartition.getParameters());
    }
    PartitionListComposingSpec composingSpec = response.getPartitionSpec().get(1).getPartitionList();
    Assert.assertNotNull("composing spec should have returned 2 partitions", composingSpec.getPartitions());
    Assert.assertEquals("composing spec should have returned 2 partitions", 2, composingSpec.getPartitionsSize());
    expectedVals.clear();
    expectedVals.add(Arrays.asList("p2", "Phoenix"));
    expectedVals.add(Arrays.asList("p2", "Seattle"));
    for (int i = 0; i < composingSpec.getPartitions().size(); i++) {
        Partition partition = composingSpec.getPartitions().get(i);
        Assert.assertEquals(2, partition.getValuesSize());
        validateList(expectedVals.get(i), partition.getValues());
        Assert.assertNull("parameters were not requested so should have been null", partition.getParameters());
    }
}