use of org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin in project drill by apache.
the class ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan method createNativeScanRel.
/**
* Helper method which creates a DrillScanRel with native Drill HiveScan.
*/
private DrillScanRel createNativeScanRel(DrillScanRel hiveScanRel, PlannerSettings settings) throws IOException {
RelDataTypeFactory typeFactory = hiveScanRel.getCluster().getTypeFactory();
HiveScan hiveScan = (HiveScan) hiveScanRel.getGroupScan();
HiveReadEntry hiveReadEntry = hiveScan.getHiveReadEntry();
Map<String, String> parameters = hiveReadEntry.getHiveTableWrapper().getParameters();
JsonScanSpec scanSpec = new JsonScanSpec(parameters.get(MAPRDB_TABLE_NAME), null, null);
List<SchemaPath> hiveScanCols = hiveScanRel.getColumns().stream().map(colNameSchemaPath -> replaceOverriddenSchemaPath(parameters, colNameSchemaPath)).collect(Collectors.toList());
// creates TupleMetadata based on Hive's schema (with optional data modes) to be used in the reader
// for the case when column type wasn't discovered
HiveToRelDataTypeConverter dataTypeConverter = new HiveToRelDataTypeConverter(typeFactory);
TupleMetadata schema = new TupleSchema();
hiveReadEntry.getTable().getColumnListsCache().getTableSchemaColumns().forEach(column -> schema.addColumn(HiveUtilities.getColumnMetadata(replaceOverriddenColumnId(parameters, column.getName()), dataTypeConverter.convertToNullableRelDataType(column))));
MapRDBFormatPluginConfig formatConfig = new MapRDBFormatPluginConfig();
formatConfig.readTimestampWithZoneOffset = settings.getOptions().getBoolean(ExecConstants.HIVE_READ_MAPRDB_JSON_TIMESTAMP_WITH_TIMEZONE_OFFSET);
formatConfig.allTextMode = settings.getOptions().getBoolean(ExecConstants.HIVE_MAPRDB_JSON_ALL_TEXT_MODE);
JsonTableGroupScan nativeMapRDBScan = new JsonTableGroupScan(hiveScan.getUserName(), hiveScan.getStoragePlugin(), // TODO: We should use Hive format plugins here, once it will be implemented. DRILL-6621
(MapRDBFormatPlugin) hiveScan.getStoragePlugin().getFormatPlugin(formatConfig), scanSpec, hiveScanCols, new MapRDBStatistics(), FileSystemMetadataProviderManager.getMetadataProviderForSchema(schema));
List<String> nativeScanColNames = hiveScanRel.getRowType().getFieldList().stream().map(field -> replaceOverriddenColumnId(parameters, field.getName())).collect(Collectors.toList());
List<RelDataType> nativeScanColTypes = hiveScanRel.getRowType().getFieldList().stream().map(RelDataTypeField::getType).collect(Collectors.toList());
RelDataType nativeScanRowType = typeFactory.createStructType(nativeScanColTypes, nativeScanColNames);
return new DrillScanRel(hiveScanRel.getCluster(), hiveScanRel.getTraitSet(), hiveScanRel.getTable(), nativeMapRDBScan, nativeScanRowType, hiveScanCols);
}
use of org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin in project drill by apache.
the class MapRDBIndexDiscover method getNativeDrillTable.
@Override
public DrillTable getNativeDrillTable(IndexDescriptor idxDescriptor) {
try {
final AbstractDbGroupScan origScan = getOriginalScan();
if (!(origScan instanceof MapRDBGroupScan)) {
return null;
}
MapRDBFormatPlugin maprFormatPlugin = ((MapRDBGroupScan) origScan).getFormatPlugin();
FileSystemPlugin fsPlugin = (FileSystemPlugin) (origScan.getStoragePlugin());
DrillFileSystem fs = ImpersonationUtil.createFileSystem(origScan.getUserName(), fsPlugin.getFsConf());
MapRDBFormatMatcher matcher = (MapRDBFormatMatcher) (maprFormatPlugin.getMatcher());
FileSelection fsSelection = deriveFSSelection(fs, idxDescriptor);
return matcher.isReadableIndex(fs, fsSelection, fsPlugin, fsPlugin.getName(), origScan.getUserName(), idxDescriptor);
} catch (Exception e) {
logger.error("Failed to get native DrillTable.", e);
}
return null;
}
Aggregations