Examples with Partition org.apache.hadoop.hive.metastore.api.Partition used on opensource projects

Example 96 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project flink by apache.

the class HiveContinuousPartitionFetcher method fetchPartitions.

@Override
@SuppressWarnings("unchecked")
public List<Tuple2<Partition, T>> fetchPartitions(Context<Partition, T> context, T previousOffset) throws Exception {
    List<Tuple2<Partition, T>> partitions = new ArrayList<>();
    List<ComparablePartitionValue> partitionValueList = context.getComparablePartitionValueList();
    for (ComparablePartitionValue<List<String>, T> partitionValue : partitionValueList) {
        T partitionOffset = partitionValue.getComparator();
        if (partitionOffset.compareTo(previousOffset) >= 0) {
            Partition partition = context.getPartition(partitionValue.getPartitionValue()).orElseThrow(() -> new IllegalArgumentException(String.format("Fetch partition fail for hive table %s.", context.getTablePath())));
            partitions.add(new Tuple2<>(partition, partitionValue.getComparator()));
        }
    }
    return partitions;
}

Example 97 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project beam by apache.

the class PartitionReaderFn method getReaderContext.

private ReaderContext getReaderContext(Read readRequest, Integer partitionIndexToRead) throws Exception {
    final List<Partition> partitions = metaStoreClient.listPartitions(readRequest.getDatabase(), readRequest.getTable(), Short.MAX_VALUE);
    final Partition partition = partitions.get(partitionIndexToRead);
    checkArgument(partition != null, "Unable to find a partition to read at index " + partitionIndexToRead);
    final int desiredSplitCount = HCatalogUtils.getSplitCount(readRequest, partition);
    final List<String> values = partition.getValues();
    final List<String> partitionCols = readRequest.getPartitionCols();
    checkArgument(values.size() == partitionCols.size(), "Number of input partitions should be equal to the values of list partition values.");
    List<String> filter = new ArrayList<>();
    for (int i = 0; i < partitionCols.size(); i++) {
        filter.add(partitionCols.get(i) + "=" + "'" + values.get(i) + "'");
    }
    final String filterString = String.join(" and ", filter);
    ReadEntity entity = new ReadEntity.Builder().withDatabase(readRequest.getDatabase()).withFilter(filterString).withTable(readRequest.getTable()).build();
    // pass the 'desired' split count as an hint to the API
    Map<String, String> configProps = new HashMap<>(readRequest.getConfigProperties());
    configProps.put(HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
    return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
}

Example 98 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project alluxio by Alluxio.

the class HiveDatabase method mountAlluxioPaths.

private PathTranslator mountAlluxioPaths(Table table, List<Partition> partitions, UdbBypassSpec bypassSpec) throws IOException {
    String tableName = table.getTableName();
    AlluxioURI ufsUri;
    AlluxioURI alluxioUri = mUdbContext.getTableLocation(tableName);
    String hiveUfsUri = table.getSd().getLocation();
    try {
        PathTranslator pathTranslator = new PathTranslator();
        if (bypassSpec.hasFullTable(tableName)) {
            pathTranslator.addMapping(hiveUfsUri, hiveUfsUri);
            return pathTranslator;
        }
        ufsUri = new AlluxioURI(table.getSd().getLocation());
        pathTranslator.addMapping(UdbUtils.mountAlluxioPath(tableName, ufsUri, alluxioUri, mUdbContext, mConfiguration), hiveUfsUri);
        for (Partition part : partitions) {
            AlluxioURI partitionUri;
            if (part.getSd() != null && part.getSd().getLocation() != null) {
                partitionUri = new AlluxioURI(part.getSd().getLocation());
                if (!mConfiguration.getBoolean(Property.ALLOW_DIFF_PART_LOC_PREFIX) && !ufsUri.isAncestorOf(partitionUri)) {
                    continue;
                }
                hiveUfsUri = part.getSd().getLocation();
                String partName = part.getValues().toString();
                try {
                    partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
                } catch (MetaException e) {
                    LOG.warn("Error making partition name for table {}, partition {}", tableName, part.getValues().toString());
                }
                if (bypassSpec.hasPartition(tableName, partName)) {
                    pathTranslator.addMapping(partitionUri.getPath(), partitionUri.getPath());
                    continue;
                }
                alluxioUri = new AlluxioURI(PathUtils.concatPath(mUdbContext.getTableLocation(tableName).getPath(), partName));
                // mount partition path if it is not already mounted as part of the table path mount
                pathTranslator.addMapping(UdbUtils.mountAlluxioPath(tableName, partitionUri, alluxioUri, mUdbContext, mConfiguration), hiveUfsUri);
            }
        }
        return pathTranslator;
    } catch (AlluxioException e) {
        throw new IOException("Failed to mount table location. tableName: " + tableName + " hiveUfsLocation: " + hiveUfsUri + " AlluxioLocation: " + alluxioUri + " error: " + e.getMessage(), e);
    }
}

Example 99 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.

the class TestEximReplicationTasks method createPtn.

private static Partition createPtn(Table t, List<String> pvals) {
    Partition ptn = new Partition();
    ptn.setDbName(t.getDbName());
    ptn.setTableName(t.getTableName());
    ptn.setValues(pvals);
    return ptn;
}

Example 100 with Partition

use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.

the class TestEximReplicationTasks method testInsert.

@Test
public void testInsert() throws HCatException {
    Table t = new Table();
    t.setDbName("testdb");
    t.setTableName("testtable");
    List<FieldSchema> pkeys = HCatSchemaUtils.getFieldSchemas(HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields());
    t.setPartitionKeys(pkeys);
    Partition p = createPtn(t, Arrays.asList("102", "lmn"));
    List<String> files = Arrays.asList("/tmp/test123");
    NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_INSERT_EVENT, msgFactory.buildInsertMessage(t.getDbName(), t.getTableName(), getPtnDesc(t, p), files).toString());
    event.setDbName(t.getDbName());
    event.setTableName(t.getTableName());
    HCatNotificationEvent hev = new HCatNotificationEvent(event);
    ReplicationTask rtask = ReplicationTask.create(client, hev);
    assertEquals(hev.toString(), rtask.getEvent().toString());
    verifyInsertReplicationTask(rtask, t, p);
}