Examples with Partition org.apache.hadoop.hive.metastore.api.Partition used on opensource projects

Example 76 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project incubator-gobblin by apache.

the class LocalHiveMetastoreTestUtils method createDummyPartition.

public org.apache.hadoop.hive.ql.metadata.Partition createDummyPartition(long createTime) {
    org.apache.hadoop.hive.ql.metadata.Partition partition = new org.apache.hadoop.hive.ql.metadata.Partition();
    Partition tPartition = new Partition();
    tPartition.setCreateTime((int) TimeUnit.SECONDS.convert(createTime, TimeUnit.MILLISECONDS));
    partition.setTPartition(tPartition);
    return partition;
}

Example 77 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project incubator-gobblin by apache.

the class HiveRetentionTest method testTimeBasedHiveRetention.

private void testTimeBasedHiveRetention(String purgedDbName, String purgedTableName, String configFileName, boolean isReplacementTest) throws Exception {
    try {
        DateTimeUtils.setCurrentMillisProvider(new FixedThreadLocalMillisProvider(FORMATTER.parseDateTime("2016-01-10-00").getMillis()));
        // Setup db, table to purge. Creating 4 partitions. 2 will be deleted and 2 will be retained
        String purgedTableSdLoc = new Path(testTempPath, purgedDbName + purgedTableName).toString();
        this.hiveMetastoreTestUtils.dropDatabaseIfExists(purgedDbName);
        final Table purgedTbl = this.hiveMetastoreTestUtils.createTestAvroTable(purgedDbName, purgedTableName, purgedTableSdLoc, ImmutableList.of("datepartition"), false);
        // Setup db, table and partitions to act as replacement partitions source
        String replacementSourceTableSdLoc = new Path(testTempPath, purgedDbName + purgedTableName + "_source").toString();
        String replacementDbName = purgedDbName + "_source";
        String replacementTableName = purgedTableName + "_source";
        this.hiveMetastoreTestUtils.dropDatabaseIfExists(replacementDbName);
        final Table replacementTbl = this.hiveMetastoreTestUtils.createTestAvroTable(replacementDbName, replacementTableName, replacementSourceTableSdLoc, ImmutableList.of("datepartition"), false);
        String deleted1 = "2016-01-01-00";
        String deleted2 = "2016-01-02-02";
        String retained1 = "2016-01-03-04";
        String retained2 = "2016-01-07-06";
        // Create partitions in table being purged
        Partition pDeleted1 = this.hiveMetastoreTestUtils.addTestPartition(purgedTbl, ImmutableList.of(deleted1), (int) System.currentTimeMillis());
        Partition pDeleted2 = this.hiveMetastoreTestUtils.addTestPartition(purgedTbl, ImmutableList.of(deleted2), (int) System.currentTimeMillis());
        Partition pRetained1 = this.hiveMetastoreTestUtils.addTestPartition(purgedTbl, ImmutableList.of(retained1), (int) System.currentTimeMillis());
        Partition pRetained2 = this.hiveMetastoreTestUtils.addTestPartition(purgedTbl, ImmutableList.of(retained2), (int) System.currentTimeMillis());
        this.fs.mkdirs(new Path(pDeleted1.getSd().getLocation()));
        this.fs.mkdirs(new Path(pDeleted2.getSd().getLocation()));
        this.fs.mkdirs(new Path(pRetained1.getSd().getLocation()));
        this.fs.mkdirs(new Path(pRetained2.getSd().getLocation()));
        // Create partitions in table that is replacement source
        Partition rReplaced1 = this.hiveMetastoreTestUtils.addTestPartition(replacementTbl, ImmutableList.of(deleted1), (int) System.currentTimeMillis());
        Partition rReplaced2 = this.hiveMetastoreTestUtils.addTestPartition(replacementTbl, ImmutableList.of(deleted2), (int) System.currentTimeMillis());
        Partition rUntouched1 = this.hiveMetastoreTestUtils.addTestPartition(replacementTbl, ImmutableList.of(retained1), (int) System.currentTimeMillis());
        Partition rUntouched2 = this.hiveMetastoreTestUtils.addTestPartition(replacementTbl, ImmutableList.of(retained2), (int) System.currentTimeMillis());
        this.fs.mkdirs(new Path(rReplaced1.getSd().getLocation()));
        this.fs.mkdirs(new Path(rReplaced2.getSd().getLocation()));
        this.fs.mkdirs(new Path(rUntouched1.getSd().getLocation()));
        this.fs.mkdirs(new Path(rUntouched2.getSd().getLocation()));
        List<Partition> pPartitions = this.hiveMetastoreTestUtils.getLocalMetastoreClient().listPartitions(purgedDbName, purgedTableName, (short) 10);
        Assert.assertEquals(pPartitions.size(), 4);
        List<Partition> rPartitions = this.hiveMetastoreTestUtils.getLocalMetastoreClient().listPartitions(replacementDbName, replacementTableName, (short) 10);
        Assert.assertEquals(rPartitions.size(), 4);
        // Run retention
        RetentionTestHelper.clean(fs, PathUtils.combinePaths(RetentionIntegrationTest.TEST_PACKAGE_RESOURCE_NAME, "testHiveTimeBasedRetention", configFileName), Optional.of(PathUtils.combinePaths(RetentionIntegrationTest.TEST_PACKAGE_RESOURCE_NAME, "testHiveTimeBasedRetention", "jobProps.properties")), testTempPath);
        pPartitions = this.hiveMetastoreTestUtils.getLocalMetastoreClient().listPartitions(purgedDbName, purgedTableName, (short) 10);
        String[] expectedRetainedPartitions;
        if (isReplacementTest) {
            // If replacement test, 2 partitions must be replaced - hence total count must be 4
            Assert.assertEquals(pPartitions.size(), 4);
            expectedRetainedPartitions = new String[] { getQlPartition(purgedTbl, pRetained1).getName(), getQlPartition(purgedTbl, pRetained2).getName(), getQlPartition(purgedTbl, pDeleted1).getName(), getQlPartition(purgedTbl, pDeleted2).getName() };
        } else {
            // If not a replacement test, 2 partitions must be purged
            Assert.assertEquals(pPartitions.size(), 2);
            expectedRetainedPartitions = new String[] { getQlPartition(purgedTbl, pRetained1).getName(), getQlPartition(purgedTbl, pRetained2).getName() };
        }
        // Check if all available partitions are that which are expected
        assertThat(FluentIterable.from(pPartitions).transform(new Function<Partition, String>() {

            @Override
            public String apply(Partition input) {
                return getQlPartition(purgedTbl, input).getName();
            }
        }).toList(), containsInAnyOrder(expectedRetainedPartitions));
        // Check that replaced partitions are pointing to correct physical location
        if (isReplacementTest) {
            for (Partition partition : pPartitions) {
                if (getQlPartition(purgedTbl, partition).getName().equalsIgnoreCase(getQlPartition(purgedTbl, pDeleted1).getName())) {
                    Assert.assertEquals(partition.getSd().getLocation(), rReplaced1.getSd().getLocation(), "Replaced partition location not updated.");
                }
                if (getQlPartition(purgedTbl, partition).getName().equalsIgnoreCase(getQlPartition(purgedTbl, pDeleted2).getName())) {
                    Assert.assertEquals(partition.getSd().getLocation(), rReplaced2.getSd().getLocation(), "Replaced partition location not updated.");
                }
            }
        }
        // Irrespective of whether it is a replacement test, purged partition directories must be deleted
        Assert.assertTrue(this.fs.exists(new Path(pRetained1.getSd().getLocation())));
        Assert.assertTrue(this.fs.exists(new Path(pRetained2.getSd().getLocation())));
        Assert.assertFalse(this.fs.exists(new Path(pDeleted1.getSd().getLocation())));
        Assert.assertFalse(this.fs.exists(new Path(pDeleted2.getSd().getLocation())));
        // Replacement source partition directories must be left untouched
        Assert.assertTrue(this.fs.exists(new Path(rReplaced1.getSd().getLocation())));
        Assert.assertTrue(this.fs.exists(new Path(rReplaced2.getSd().getLocation())));
        Assert.assertTrue(this.fs.exists(new Path(rUntouched1.getSd().getLocation())));
        Assert.assertTrue(this.fs.exists(new Path(rUntouched2.getSd().getLocation())));
    } finally {
        DateTimeUtils.setCurrentMillisSystem();
    }
}

Example 78 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project drill by axbaretto.

the class HiveScan method getSpecificScan.

@Override
public SubScan getSpecificScan(final int minorFragmentId) throws ExecutionSetupException {
    try {
        final List<LogicalInputSplit> splits = mappings.get(minorFragmentId);
        List<HivePartitionWrapper> parts = Lists.newArrayList();
        final List<List<String>> encodedInputSplits = Lists.newArrayList();
        final List<String> splitTypes = Lists.newArrayList();
        for (final LogicalInputSplit split : splits) {
            final Partition splitPartition = split.getPartition();
            if (splitPartition != null) {
                HiveTableWithColumnCache table = hiveReadEntry.getTable();
                parts.add(createPartitionWithSpecColumns(new HiveTableWithColumnCache(table, new ColumnListsCache(table)), splitPartition));
            }
            encodedInputSplits.add(split.serialize());
            splitTypes.add(split.getType());
        }
        if (parts.size() <= 0) {
            parts = null;
        }
        final HiveReadEntry subEntry = new HiveReadEntry(hiveReadEntry.getTableWrapper(), parts);
        return new HiveSubScan(getUserName(), encodedInputSplits, subEntry, splitTypes, columns, hiveStoragePlugin);
    } catch (IOException | ReflectiveOperationException e) {
        throw new ExecutionSetupException(e);
    }
}

Example 79 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project drill by apache.

the class HiveUtilities method getPartitionMetadata.

/**
 * Wrapper around {@code MetaStoreUtils#getPartitionMetadata(org.apache.hadoop.hive.metastore.api.Partition, Table)}
 * which also adds parameters from table to properties returned by that method.
 *
 * @param partition the source of partition level parameters
 * @param table     the source of table level parameters
 * @return properties
 */
public static Properties getPartitionMetadata(final HivePartition partition, final HiveTableWithColumnCache table) {
    restoreColumns(table, partition);
    try {
        Properties properties = new org.apache.hadoop.hive.ql.metadata.Partition(new org.apache.hadoop.hive.ql.metadata.Table(table), partition).getMetadataFromPartitionSchema();
        // SerDe expects properties from Table, but above call doesn't add Table properties.
        // Include Table properties in final list in order to not to break SerDes that depend on
        // Table properties. For example AvroSerDe gets the schema from properties (passed as second argument)
        table.getParameters().entrySet().stream().filter(e -> e.getKey() != null && e.getValue() != null).forEach(e -> properties.put(e.getKey(), e.getValue()));
        return properties;
    } catch (HiveException e) {
        throw new DrillRuntimeException(e);
    }
}

Example 80 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project drill by apache.

the class HiveScan method getSpecificScan.

@Override
public SubScan getSpecificScan(final int minorFragmentId) throws ExecutionSetupException {
    try {
        final List<LogicalInputSplit> splits = mappings.get(minorFragmentId);
        List<HivePartitionWrapper> parts = new ArrayList<>();
        final List<List<String>> encodedInputSplits = new ArrayList<>();
        final List<String> splitTypes = new ArrayList<>();
        for (final LogicalInputSplit split : splits) {
            final Partition splitPartition = split.getPartition();
            if (splitPartition != null) {
                HiveTableWithColumnCache table = hiveReadEntry.getTable();
                parts.add(createPartitionWithSpecColumns(new HiveTableWithColumnCache(table, new ColumnListsCache(table)), splitPartition));
            }
            encodedInputSplits.add(split.serialize());
            splitTypes.add(split.getType());
        }
        if (parts.size() <= 0) {
            parts = null;
        }
        final HiveReadEntry subEntry = new HiveReadEntry(hiveReadEntry.getTableWrapper(), parts);
        return new HiveSubScan(getUserName(), encodedInputSplits, subEntry, splitTypes, columns, hiveStoragePlugin, confProperties);
    } catch (IOException | ReflectiveOperationException e) {
        throw new ExecutionSetupException(e);
    }
}