Examples with DrillScanRel org.apache.drill.exec.planner.logical.DrillScanRel used on opensource projects

Example 11 with DrillScanRel

use of org.apache.drill.exec.planner.logical.DrillScanRel in project drill by apache.

the class ConvertHiveParquetScanToDrillParquetScan method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    try {
        final DrillScanRel hiveScanRel = (DrillScanRel) call.rel(0);
        final HiveScan hiveScan = (HiveScan) hiveScanRel.getGroupScan();
        final PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
        final String partitionColumnLabel = settings.getFsPartitionColumnLabel();
        final Table hiveTable = hiveScan.hiveReadEntry.getTable();
        checkForUnsupportedDataTypes(hiveTable);
        final Map<String, String> partitionColMapping = getPartitionColMapping(hiveTable, partitionColumnLabel);
        final DrillScanRel nativeScanRel = createNativeScanRel(partitionColMapping, hiveScanRel);
        if (hiveScanRel.getRowType().getFieldCount() == 0) {
            call.transformTo(nativeScanRel);
        } else {
            final DrillProjectRel projectRel = createProjectRel(hiveScanRel, partitionColMapping, nativeScanRel);
            call.transformTo(projectRel);
        }
    } catch (final Exception e) {
        logger.warn("Failed to convert HiveScan to HiveDrillNativeParquetScan", e);
    }
}

Example 12 with DrillScanRel

use of org.apache.drill.exec.planner.logical.DrillScanRel in project drill by apache.

the class ConvertHiveParquetScanToDrillParquetScan method matches.

/**
   * Rule is matched when all of the following match:
   * 1) GroupScan in given DrillScalRel is an {@link HiveScan}
   * 2) {@link HiveScan} is not already rewritten using Drill's native readers
   * 3) InputFormat in Hive table metadata and all partitions metadata contains the same value
   *    {@link MapredParquetInputFormat}
   * 4) No error occurred while checking for the above conditions. An error is logged as warning.
   *
   * @param call
   * @return True if the rule can be applied. False otherwise
   */
@Override
public boolean matches(RelOptRuleCall call) {
    final DrillScanRel scanRel = (DrillScanRel) call.rel(0);
    if (!(scanRel.getGroupScan() instanceof HiveScan) || ((HiveScan) scanRel.getGroupScan()).isNativeReader()) {
        return false;
    }
    final HiveScan hiveScan = (HiveScan) scanRel.getGroupScan();
    final HiveConf hiveConf = hiveScan.getHiveConf();
    final HiveTableWithColumnCache hiveTable = hiveScan.hiveReadEntry.getTable();
    final Class<? extends InputFormat<?, ?>> tableInputFormat = getInputFormatFromSD(HiveUtilities.getTableMetadata(hiveTable), hiveScan.hiveReadEntry, hiveTable.getSd(), hiveConf);
    if (tableInputFormat == null || !tableInputFormat.equals(MapredParquetInputFormat.class)) {
        return false;
    }
    final List<HivePartitionWrapper> partitions = hiveScan.hiveReadEntry.getHivePartitionWrappers();
    if (partitions == null) {
        return true;
    }
    final List<FieldSchema> tableSchema = hiveTable.getSd().getCols();
    // Make sure all partitions have the same input format as the table input format
    for (HivePartitionWrapper partition : partitions) {
        final StorageDescriptor partitionSD = partition.getPartition().getSd();
        Class<? extends InputFormat<?, ?>> inputFormat = getInputFormatFromSD(HiveUtilities.getPartitionMetadata(partition.getPartition(), hiveTable), hiveScan.hiveReadEntry, partitionSD, hiveConf);
        if (inputFormat == null || !inputFormat.equals(tableInputFormat)) {
            return false;
        }
        // possible types. Drill doesn't have the similar set of methods yet.
        if (!partitionSD.getCols().equals(tableSchema)) {
            logger.debug("Partitions schema is different from table schema. Currently native reader conversion can't " + "handle schema difference between partitions and table");
            return false;
        }
    }
    return true;
}