Examples with Partition org.apache.hadoop.hive.metastore.api.Partition used on opensource projects

Example 46 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.

the class TestHBaseStore method dropPartition.

@Test
public void dropPartition() throws Exception {
    String tableName = "myparttable2";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema("pc", "string", ""));
    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    store.createTable(table);
    List<String> vals = Arrays.asList("fred");
    StorageDescriptor psd = new StorageDescriptor(sd);
    psd.setLocation("file:/tmp/pc=fred");
    Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd, emptyParameters);
    store.addPartition(part);
    Assert.assertNotNull(store.getPartition(DB, tableName, vals));
    store.dropPartition(DB, tableName, vals);
    thrown.expect(NoSuchObjectException.class);
    store.getPartition(DB, tableName, vals);
}

Example 47 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.

the class TestHBaseAggregateStatsNDVUniformDist method TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong.

@Test
public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
    String dbName = "default";
    String tableName = "snp";
    long now = System.currentTimeMillis();
    List<FieldSchema> cols = new ArrayList<>();
    cols.add(new FieldSchema("col5_long", "long", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, Collections.<String, String>emptyMap());
    List<FieldSchema> partCols = new ArrayList<>();
    partCols.add(new FieldSchema("ds", "string", ""));
    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols, Collections.<String, String>emptyMap(), null, null, null);
    store.createTable(table);
    List<List<String>> partVals = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        List<String> partVal = Arrays.asList("" + i);
        partVals.add(partVal);
        StorageDescriptor psd = new StorageDescriptor(sd);
        psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
        Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd, Collections.<String, String>emptyMap());
        store.addPartition(part);
        if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
            ColumnStatistics cs = new ColumnStatistics();
            ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
            desc.setLastAnalyzed(now);
            desc.setPartName("ds=" + partVal);
            cs.setStatsDesc(desc);
            ColumnStatisticsObj obj = new ColumnStatisticsObj();
            obj.setColName("col5_long");
            obj.setColType("long");
            ColumnStatisticsData data = new ColumnStatisticsData();
            LongColumnStatsData dcsd = new LongColumnStatsData();
            dcsd.setHighValue(1000 + i);
            dcsd.setLowValue(-1000 - i);
            dcsd.setNumNulls(i);
            dcsd.setNumDVs(10 * i + 1);
            dcsd.setBitVectors(bitVectors[i / 5]);
            data.setLongStats(dcsd);
            obj.setStatsData(data);
            cs.addToStatsObj(obj);
            store.updatePartitionColumnStatistics(cs, partVal);
        }
    }
    Checker statChecker = new Checker() {

        @Override
        public void checkStats(AggrStats aggrStats) throws Exception {
            Assert.assertEquals(6, aggrStats.getPartsFound());
            Assert.assertEquals(1, aggrStats.getColStatsSize());
            ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
            Assert.assertEquals("col5_long", cso.getColName());
            Assert.assertEquals("long", cso.getColType());
            LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
            Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
            Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
            Assert.assertEquals(40, lcsd.getNumNulls());
            Assert.assertEquals(12, lcsd.getNumDVs());
        }
    };
    List<String> partNames = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        partNames.add("ds=" + i);
    }
    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames, Arrays.asList("col5_long"));
    statChecker.checkStats(aggrStats);
}

Example 48 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project presto by prestodb.

the class InMemoryHiveMetastore method listAllDataPaths.

private static List<String> listAllDataPaths(HiveMetastore metastore, String schemaName, String tableName) {
    ImmutableList.Builder<String> locations = ImmutableList.builder();
    Table table = metastore.getTable(schemaName, tableName).get();
    if (table.getSd().getLocation() != null) {
        // For unpartitioned table, there should be nothing directly under this directory.
        // But including this location in the set makes the directory content assert more
        // extensive, which is desirable.
        locations.add(table.getSd().getLocation());
    }
    Optional<List<String>> partitionNames = metastore.getPartitionNames(schemaName, tableName);
    if (partitionNames.isPresent()) {
        metastore.getPartitionsByNames(schemaName, tableName, partitionNames.get()).stream().map(partition -> partition.getSd().getLocation()).filter(location -> !location.startsWith(table.getSd().getLocation())).forEach(locations::add);
    }
    return locations.build();
}

Example 49 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project presto by prestodb.

the class InMemoryHiveMetastore method getPartition.

@Override
public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues) {
    PartitionName name = PartitionName.partition(databaseName, tableName, partitionValues);
    Partition partition = partitions.get(name);
    if (partition == null) {
        return Optional.empty();
    }
    return Optional.of(partition.deepCopy());
}

Example 50 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project presto by prestodb.

the class InMemoryHiveMetastore method getPartitionsByNames.

@Override
public synchronized List<Partition> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) {
    ImmutableList.Builder<Partition> builder = ImmutableList.builder();
    for (String name : partitionNames) {
        PartitionName partitionName = PartitionName.partition(databaseName, tableName, name);
        Partition partition = partitions.get(partitionName);
        if (partition == null) {
            return ImmutableList.of();
        }
        builder.add(partition.deepCopy());
    }
    return builder.build();
}