Examples with HivePartitionWrapper org.apache.drill.exec.store.hive.HiveTableWrapper.HivePartitionWrapper used on opensource projects

Example 1 with HivePartitionWrapper

use of org.apache.drill.exec.store.hive.HiveTableWrapper.HivePartitionWrapper in project drill by apache.

the class HiveScan method getSpecificScan.

@Override
public SubScan getSpecificScan(final int minorFragmentId) throws ExecutionSetupException {
    try {
        final List<InputSplitWrapper> splits = mappings.get(minorFragmentId);
        List<HivePartitionWrapper> parts = Lists.newArrayList();
        final List<String> encodedInputSplits = Lists.newArrayList();
        final List<String> splitTypes = Lists.newArrayList();
        for (final InputSplitWrapper split : splits) {
            final Partition splitPartition = split.getPartition();
            if (splitPartition != null) {
                HiveTableWithColumnCache table = hiveReadEntry.getTable();
                parts.add(createPartitionWithSpecColumns(new HiveTableWithColumnCache(table, new ColumnListsCache(table)), splitPartition));
            }
            encodedInputSplits.add(serializeInputSplit(split.getSplit()));
            splitTypes.add(split.getSplit().getClass().getName());
        }
        if (parts.size() <= 0) {
            parts = null;
        }
        final HiveReadEntry subEntry = new HiveReadEntry(hiveReadEntry.getTableWrapper(), parts);
        return new HiveSubScan(getUserName(), encodedInputSplits, subEntry, splitTypes, columns, storagePlugin);
    } catch (IOException | ReflectiveOperationException e) {
        throw new ExecutionSetupException(e);
    }
}

Example 2 with HivePartitionWrapper

use of org.apache.drill.exec.store.hive.HiveTableWrapper.HivePartitionWrapper in project drill by apache.

the class HiveScan method toString.

@Override
public String toString() {
    List<HivePartitionWrapper> partitions = hiveReadEntry.getHivePartitionWrappers();
    int numPartitions = partitions == null ? 0 : partitions.size();
    return "HiveScan [table=" + hiveReadEntry.getHiveTableWrapper() + ", columns=" + columns + ", numPartitions=" + numPartitions + ", partitions= " + partitions + ", inputDirectories=" + metadataProvider.getInputDirectories(hiveReadEntry) + "]";
}

Example 3 with HivePartitionWrapper

use of org.apache.drill.exec.store.hive.HiveTableWrapper.HivePartitionWrapper in project drill by apache.

the class ConvertHiveParquetScanToDrillParquetScan method matches.

/**
   * Rule is matched when all of the following match:
   * 1) GroupScan in given DrillScalRel is an {@link HiveScan}
   * 2) {@link HiveScan} is not already rewritten using Drill's native readers
   * 3) InputFormat in Hive table metadata and all partitions metadata contains the same value
   *    {@link MapredParquetInputFormat}
   * 4) No error occurred while checking for the above conditions. An error is logged as warning.
   *
   * @param call
   * @return True if the rule can be applied. False otherwise
   */
@Override
public boolean matches(RelOptRuleCall call) {
    final DrillScanRel scanRel = (DrillScanRel) call.rel(0);
    if (!(scanRel.getGroupScan() instanceof HiveScan) || ((HiveScan) scanRel.getGroupScan()).isNativeReader()) {
        return false;
    }
    final HiveScan hiveScan = (HiveScan) scanRel.getGroupScan();
    final HiveConf hiveConf = hiveScan.getHiveConf();
    final HiveTableWithColumnCache hiveTable = hiveScan.hiveReadEntry.getTable();
    final Class<? extends InputFormat<?, ?>> tableInputFormat = getInputFormatFromSD(HiveUtilities.getTableMetadata(hiveTable), hiveScan.hiveReadEntry, hiveTable.getSd(), hiveConf);
    if (tableInputFormat == null || !tableInputFormat.equals(MapredParquetInputFormat.class)) {
        return false;
    }
    final List<HivePartitionWrapper> partitions = hiveScan.hiveReadEntry.getHivePartitionWrappers();
    if (partitions == null) {
        return true;
    }
    final List<FieldSchema> tableSchema = hiveTable.getSd().getCols();
    // Make sure all partitions have the same input format as the table input format
    for (HivePartitionWrapper partition : partitions) {
        final StorageDescriptor partitionSD = partition.getPartition().getSd();
        Class<? extends InputFormat<?, ?>> inputFormat = getInputFormatFromSD(HiveUtilities.getPartitionMetadata(partition.getPartition(), hiveTable), hiveScan.hiveReadEntry, partitionSD, hiveConf);
        if (inputFormat == null || !inputFormat.equals(tableInputFormat)) {
            return false;
        }
        // possible types. Drill doesn't have the similar set of methods yet.
        if (!partitionSD.getCols().equals(tableSchema)) {
            logger.debug("Partitions schema is different from table schema. Currently native reader conversion can't " + "handle schema difference between partitions and table");
            return false;
        }
    }
    return true;
}