use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.
the class KuduScanBatchCreator method getBatch.
@Override
public ScanBatch getBatch(ExecutorFragmentContext context, KuduSubScan subScan, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = new LinkedList<>();
List<SchemaPath> columns = null;
for (KuduSubScan.KuduSubScanSpec scanSpec : subScan.getTabletScanSpecList()) {
try {
if ((columns = subScan.getColumns()) == null) {
columns = GroupScan.ALL_COLUMNS;
}
readers.add(new KuduRecordReader(subScan.getStorageEngine().getClient(), scanSpec, columns));
} catch (Exception e1) {
throw new ExecutionSetupException(e1);
}
}
return new ScanBatch(subScan, context, readers);
}
use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.
the class MapRDBScanBatchCreator method getBatch.
@Override
public ScanBatch getBatch(ExecutorFragmentContext context, MapRDBSubScan subScan, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = new LinkedList<>();
for (MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()) {
try {
if (BinaryTableGroupScan.TABLE_BINARY.equals(subScan.getTableType())) {
readers.add(new HBaseRecordReader(subScan.getFormatPlugin().getConnection(), getHBaseSubScanSpec(scanSpec), subScan.getColumns()));
} else {
readers.add(new MaprDBJsonRecordReader(scanSpec, subScan.getFormatPlugin(), subScan.getColumns(), context, subScan.getMaxRecordsToRead(), subScan.getSchema()));
}
} catch (Exception e) {
throw new ExecutionSetupException(e);
}
}
return new ScanBatch(subScan, context, readers);
}
use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.
the class MapRDBRestrictedScanBatchCreator method getBatch.
@Override
public ScanBatch getBatch(ExecutorFragmentContext context, RestrictedMapRDBSubScan subScan, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = Lists.newArrayList();
for (MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()) {
try {
readers.add(new RestrictedJsonRecordReader(scanSpec, subScan.getFormatPlugin(), subScan.getColumns(), context, subScan.getMaxRecordsToRead(), subScan.getSchema()));
} catch (Exception e1) {
throw new ExecutionSetupException(e1);
}
}
return new ScanBatch(subScan, context, readers, true);
}
use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.
the class HBaseScanBatchCreator method getBatch.
@Override
public ScanBatch getBatch(ExecutorFragmentContext context, HBaseSubScan subScan, List<RecordBatch> children) throws ExecutionSetupException {
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = new LinkedList<>();
List<SchemaPath> columns = null;
for (HBaseSubScan.HBaseSubScanSpec scanSpec : subScan.getRegionScanSpecList()) {
try {
if ((columns = subScan.getColumns()) == null) {
columns = GroupScan.ALL_COLUMNS;
}
readers.add(new HBaseRecordReader(subScan.getStorageEngine().getConnection(), scanSpec, columns));
} catch (Exception e1) {
throw new ExecutionSetupException(e1);
}
}
return new ScanBatch(subScan, context, readers);
}
use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.
the class DrillParquetReader method setup.
@Override
public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
try {
this.operatorContext = context;
schema = footer.getFileMetaData().getSchema();
MessageType projection;
final List<SchemaPath> columnsNotFound = new ArrayList<>(getColumns().size());
if (isStarQuery()) {
projection = schema;
} else {
projection = getProjection(schema, getColumns(), columnsNotFound);
if (projection == null) {
projection = schema;
}
if (!columnsNotFound.isEmpty()) {
nullFilledVectors = new ArrayList<>(columnsNotFound.size());
for (SchemaPath col : columnsNotFound) {
// col.toExpr() is used here as field name since we don't want to see these fields in the existing maps
nullFilledVectors.add(output.addField(MaterializedField.create(col.toExpr(), OPTIONAL_INT), NullableIntVector.class));
}
noColumnsFound = columnsNotFound.size() == getColumns().size();
}
}
logger.debug("Requesting schema {}", projection);
if (!noColumnsFound) {
// Discard the columns not found in the schema when create DrillParquetRecordMaterializer, since they have been added to output already.
@SuppressWarnings("unchecked") Collection<SchemaPath> columns = columnsNotFound.isEmpty() ? getColumns() : CollectionUtils.subtract(getColumns(), columnsNotFound);
recordMaterializer = new DrillParquetRecordMaterializer(output, projection, columns, fragmentContext.getOptions(), containsCorruptedDates);
}
if (numRecordsToRead == 0 || noColumnsFound) {
// no need to init readers
return;
}
ColumnIOFactory factory = new ColumnIOFactory(false);
MessageColumnIO columnIO = factory.getColumnIO(projection, schema);
BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());
Map<ColumnPath, ColumnChunkMetaData> paths = blockMetaData.getColumns().stream().collect(Collectors.toMap(ColumnChunkMetaData::getPath, Function.identity(), (o, n) -> n));
BufferAllocator allocator = operatorContext.getAllocator();
CompressionCodecFactory ccf = DrillCompressionCodecFactory.createDirectCodecFactory(drillFileSystem.getConf(), new ParquetDirectByteBufferAllocator(allocator), 0);
pageReadStore = new ColumnChunkIncReadStore(numRecordsToRead, ccf, allocator, drillFileSystem, entry.getPath());
for (String[] path : schema.getPaths()) {
Type type = schema.getType(path);
if (type.isPrimitive()) {
ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
pageReadStore.addColumn(schema.getColumnDescription(path), md);
}
}
recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);
} catch (Exception e) {
throw handleAndRaise("Failure in setting up reader", e);
}
}
Aggregations