Examples with PartitionSpec org.apache.hadoop.hive.metastore.api.PartitionSpec used on opensource projects

Example 6 with PartitionSpec

use of org.apache.hadoop.hive.metastore.api.PartitionSpec in project hive by apache.

the class TestHiveMetaStorePartitionSpecs method testFetchingPartitionsWithDifferentSchemas.

/**
 * Test to confirm that Partition-grouping behaves correctly when Table-schemas evolve.
 * Partitions must be grouped by location and schema.
 */
@Test
public void testFetchingPartitionsWithDifferentSchemas() {
    try {
        // Create source table.
        HiveMetaStoreClient hmsc = new HiveMetaStoreClient(conf);
        clearAndRecreateDB(hmsc);
        createTable(hmsc, true);
        Table table = hmsc.getTable(dbName, tableName);
        populatePartitions(hmsc, table, // Blurb list.
        Arrays.asList("isLocatedInTablePath", "isLocatedOutsideTablePath"));
        // Modify table schema. Add columns.
        List<FieldSchema> fields = table.getSd().getCols();
        fields.add(new FieldSchema("goo", "string", "Entirely new column. Doesn't apply to older partitions."));
        table.getSd().setCols(fields);
        hmsc.alter_table(dbName, tableName, table);
        // Check that the change stuck.
        table = hmsc.getTable(dbName, tableName);
        Assert.assertEquals("Unexpected number of table columns.", 3, table.getSd().getColsSize());
        // Add partitions with new schema.
        // Mark Partitions with new schema with different blurb.
        populatePartitions(hmsc, table, Arrays.asList("hasNewColumn"));
        // Retrieve *all* partitions from the table.
        PartitionSpecProxy partitionSpecProxy = hmsc.listPartitionSpecs(dbName, tableName, -1);
        Assert.assertEquals("Unexpected number of partitions.", nDates * 3, partitionSpecProxy.size());
        // Confirm grouping.
        Assert.assertTrue("Unexpected type of PartitionSpecProxy.", partitionSpecProxy instanceof CompositePartitionSpecProxy);
        CompositePartitionSpecProxy compositePartitionSpecProxy = (CompositePartitionSpecProxy) partitionSpecProxy;
        List<PartitionSpec> partitionSpecs = compositePartitionSpecProxy.toPartitionSpec();
        Assert.assertTrue("PartitionSpec[0] should have been a SharedSDPartitionSpec.", partitionSpecs.get(0).isSetSharedSDPartitionSpec());
        Assert.assertEquals("PartitionSpec[0] should use the table-path as the common root location. ", table.getSd().getLocation(), partitionSpecs.get(0).getRootPath());
        Assert.assertTrue("PartitionSpec[1] should have been a SharedSDPartitionSpec.", partitionSpecs.get(1).isSetSharedSDPartitionSpec());
        Assert.assertEquals("PartitionSpec[1] should use the table-path as the common root location. ", table.getSd().getLocation(), partitionSpecs.get(1).getRootPath());
        Assert.assertTrue("PartitionSpec[2] should have been a ListComposingPartitionSpec.", partitionSpecs.get(2).isSetPartitionList());
        // Categorize the partitions returned, and confirm that all partitions are accounted for.
        PartitionSpecProxy.PartitionIterator iterator = partitionSpecProxy.getPartitionIterator();
        Map<String, List<Partition>> blurbToPartitionList = new HashMap<>(3);
        while (iterator.hasNext()) {
            Partition partition = iterator.next();
            String blurb = partition.getValues().get(1);
            if (!blurbToPartitionList.containsKey(blurb)) {
                blurbToPartitionList.put(blurb, new ArrayList<>(nDates));
            }
            blurbToPartitionList.get(blurb).add(partition);
        }
        // and must have locations outside the table directory.
        for (Partition partition : blurbToPartitionList.get("isLocatedOutsideTablePath")) {
            Assert.assertEquals("Unexpected number of columns.", 2, partition.getSd().getCols().size());
            Assert.assertEquals("Unexpected first column.", "foo", partition.getSd().getCols().get(0).getName());
            Assert.assertEquals("Unexpected second column.", "bar", partition.getSd().getCols().get(1).getName());
            String partitionLocation = partition.getSd().getLocation();
            String tableLocation = table.getSd().getLocation();
            Assert.assertTrue("Unexpected partition location: " + partitionLocation + ". " + "Partition should have been outside table location: " + tableLocation, !partitionLocation.startsWith(tableLocation));
        }
        // and must have locations within the table directory.
        for (Partition partition : blurbToPartitionList.get("isLocatedInTablePath")) {
            Assert.assertEquals("Unexpected number of columns.", 2, partition.getSd().getCols().size());
            Assert.assertEquals("Unexpected first column.", "foo", partition.getSd().getCols().get(0).getName());
            Assert.assertEquals("Unexpected second column.", "bar", partition.getSd().getCols().get(1).getName());
            String partitionLocation = partition.getSd().getLocation();
            String tableLocation = table.getSd().getLocation();
            Assert.assertTrue("Unexpected partition location: " + partitionLocation + ". " + "Partition should have been within table location: " + tableLocation, partitionLocation.startsWith(tableLocation));
        }
        // and must have 3 columns. Also, the partition locations must lie within the table directory.
        for (Partition partition : blurbToPartitionList.get("hasNewColumn")) {
            Assert.assertEquals("Unexpected number of columns.", 3, partition.getSd().getCols().size());
            Assert.assertEquals("Unexpected first column.", "foo", partition.getSd().getCols().get(0).getName());
            Assert.assertEquals("Unexpected second column.", "bar", partition.getSd().getCols().get(1).getName());
            Assert.assertEquals("Unexpected third column.", "goo", partition.getSd().getCols().get(2).getName());
            String partitionLocation = partition.getSd().getLocation();
            String tableLocation = table.getSd().getLocation();
            Assert.assertTrue("Unexpected partition location: " + partitionLocation + ". " + "Partition should have been within table location: " + tableLocation, partitionLocation.startsWith(tableLocation));
        }
    } catch (Throwable t) {
        LOG.error("Unexpected Exception!", t);
        t.printStackTrace();
        Assert.assertTrue("Unexpected Exception!", false);
    }
}

Example 7 with PartitionSpec

use of org.apache.hadoop.hive.metastore.api.PartitionSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionAllSingleValuedFields.

@Test
public void testPartitionProjectionAllSingleValuedFields() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Arrays.asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib");
    // TODO directSQL does not support serdeType, serializerClass and deserializerClass in serdeInfo
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
}

Example 8 with PartitionSpec

use of org.apache.hadoop.hive.metastore.api.PartitionSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testPartitionProjectionAllMultiValuedFields.

@Test
public void testPartitionProjectionAllMultiValuedFields() throws Throwable {
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Arrays.asList("values", "parameters", "sd.cols", "sd.bucketCols", "sd.sortCols", "sd.parameters", "sd.skewedInfo", "sd.serdeInfo.parameters");
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    Assert.assertEquals(origPartitions.size(), partitionSpecWithSharedSD.getPartitions().size());
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    for (int i = 0; i < origPartitions.size(); i++) {
        Partition origPartition = origPartitions.get(i);
        PartitionWithoutSD retPartition = partitionSpecWithSharedSD.getPartitions().get(i);
        for (String projectedField : projectedFields) {
            switch(projectedField) {
                case "values":
                    validateList(origPartition.getValues(), retPartition.getValues());
                    break;
                case "parameters":
                    validateMap(origPartition.getParameters(), retPartition.getParameters());
                    break;
                case "sd.cols":
                    validateList(origPartition.getSd().getCols(), sharedSD.getCols());
                    break;
                case "sd.bucketCols":
                    validateList(origPartition.getSd().getBucketCols(), sharedSD.getBucketCols());
                    break;
                case "sd.sortCols":
                    validateList(origPartition.getSd().getSortCols(), sharedSD.getSortCols());
                    break;
                case "sd.parameters":
                    validateMap(origPartition.getSd().getParameters(), sharedSD.getParameters());
                    break;
                case "sd.skewedInfo":
                    if (!origPartition.getSd().getSkewedInfo().getSkewedColNames().isEmpty()) {
                        validateList(origPartition.getSd().getSkewedInfo().getSkewedColNames(), sharedSD.getSkewedInfo().getSkewedColNames());
                    }
                    if (!origPartition.getSd().getSkewedInfo().getSkewedColValues().isEmpty()) {
                        for (int i1 = 0; i1 < origPartition.getSd().getSkewedInfo().getSkewedColValuesSize(); i1++) {
                            validateList(origPartition.getSd().getSkewedInfo().getSkewedColValues().get(i1), sharedSD.getSkewedInfo().getSkewedColValues().get(i1));
                        }
                    }
                    if (!origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps().isEmpty()) {
                        validateMap(origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps(), sharedSD.getSkewedInfo().getSkewedColValueLocationMaps());
                    }
                    break;
                case "sd.serdeInfo.parameters":
                    validateMap(origPartition.getSd().getSerdeInfo().getParameters(), sharedSD.getSerdeInfo().getParameters());
                    break;
                default:
                    throw new IllegalArgumentException("Invalid field " + projectedField);
            }
        }
    }
}

Example 9 with PartitionSpec

use of org.apache.hadoop.hive.metastore.api.PartitionSpec in project hive by apache.

the class TestGetPartitionsUsingProjectionAndFilterSpecs method testProjectionUsingJDO.

@Test
public void testProjectionUsingJDO() throws Throwable {
    // disable direct SQL to make sure
    client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "false");
    GetPartitionsRequest request = getGetPartitionsRequest();
    GetProjectionsSpec projectSpec = request.getProjectionSpec();
    List<String> projectedFields = Collections.singletonList("sd.location");
    projectSpec.setFieldList(projectedFields);
    GetPartitionsResponse response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    PartitionSpec partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
    // set all the single-valued fields and try using JDO
    request = getGetPartitionsRequest();
    projectSpec = request.getProjectionSpec();
    projectedFields = Arrays.asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.serdeType", "sd.serdeInfo.serializerClass", "sd.serdeInfo.deserializerClass");
    projectSpec.setFieldList(projectedFields);
    response = client.getPartitionsWithSpecs(request);
    Assert.assertEquals(1, response.getPartitionSpec().size());
    partitionSpec = response.getPartitionSpec().get(0);
    Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName());
    Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName());
    partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec();
    sharedSD = partitionSpecWithSharedSD.getSd();
    Assert.assertNotNull(sharedSD);
    partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
    Assert.assertNotNull(partitionWithoutSDS);
    Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
    comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0);
}

Example 10 with PartitionSpec

use of org.apache.hadoop.hive.metastore.api.PartitionSpec in project hive by apache.

the class TestAddPartitionsFromPartSpec method testAddPartitionSpecWithSharedSDUpperCaseDBAndTableName.

@Test
public void testAddPartitionSpecWithSharedSDUpperCaseDBAndTableName() throws Exception {
    Table table = createTable();
    PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2013"), 1);
    List<PartitionWithoutSD> partitions = Lists.newArrayList(partition);
    String location = table.getSd().getLocation() + "/sharedSDTest/";
    PartitionSpec partitionSpec = new PartitionSpec();
    partitionSpec.setDbName(DB_NAME.toUpperCase());
    partitionSpec.setTableName(TABLE_NAME.toUpperCase());
    PartitionSpecWithSharedSD partitionList = new PartitionSpecWithSharedSD();
    partitionList.setPartitions(partitions);
    partitionList.setSd(buildSD(location));
    partitionSpec.setSharedSDPartitionSpec(partitionList);
    PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partitionSpec);
    client.add_partitions_pspec(partitionSpecProxy);
    Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=2013");
    Assert.assertNotNull(part);
    Assert.assertEquals(DB_NAME, part.getDbName());
    Assert.assertEquals(TABLE_NAME, part.getTableName());
    Assert.assertEquals(metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/partwithoutsd1", part.getSd().getLocation());
}