Examples with ExecutionSetupException org.apache.drill.common.exceptions.ExecutionSetupException used on opensource projects

Example 76 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class EasyFormatPlugin method getReaderBatch.

@SuppressWarnings("resource")
CloseableRecordBatch getReaderBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException {
    final ColumnExplorer columnExplorer = new ColumnExplorer(context.getOptions(), scan.getColumns());
    if (!columnExplorer.isStarQuery()) {
        scan = new EasySubScan(scan.getUserName(), scan.getWorkUnits(), scan.getFormatPlugin(), columnExplorer.getTableColumns(), scan.getSelectionRoot());
        scan.setOperatorId(scan.getOperatorId());
    }
    OperatorContext oContext = context.newOperatorContext(scan);
    final DrillFileSystem dfs;
    try {
        dfs = oContext.newFileSystem(fsConf);
    } catch (IOException e) {
        throw new ExecutionSetupException(String.format("Failed to create FileSystem: %s", e.getMessage()), e);
    }
    List<RecordReader> readers = new LinkedList<>();
    List<Map<String, String>> implicitColumns = Lists.newArrayList();
    Map<String, String> mapWithMaxColumns = Maps.newLinkedHashMap();
    for (FileWork work : scan.getWorkUnits()) {
        RecordReader recordReader = getRecordReader(context, dfs, work, scan.getColumns(), scan.getUserName());
        readers.add(recordReader);
        Map<String, String> implicitValues = columnExplorer.populateImplicitColumns(work, scan.getSelectionRoot());
        implicitColumns.add(implicitValues);
        if (implicitValues.size() > mapWithMaxColumns.size()) {
            mapWithMaxColumns = implicitValues;
        }
    }
    // all readers should have the same number of implicit columns, add missing ones with value null
    Map<String, String> diff = Maps.transformValues(mapWithMaxColumns, Functions.constant((String) null));
    for (Map<String, String> map : implicitColumns) {
        map.putAll(Maps.difference(map, diff).entriesOnlyOnRight());
    }
    return new ScanBatch(context, oContext, readers, implicitColumns);
}

Example 77 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class MapRDBScanBatchCreator method getBatch.

@Override
public ScanBatch getBatch(ExecutorFragmentContext context, MapRDBSubScan subScan, List<RecordBatch> children) throws ExecutionSetupException {
    Preconditions.checkArgument(children.isEmpty());
    List<RecordReader> readers = new LinkedList<>();
    for (MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()) {
        try {
            if (BinaryTableGroupScan.TABLE_BINARY.equals(subScan.getTableType())) {
                readers.add(new HBaseRecordReader(subScan.getFormatPlugin().getConnection(), getHBaseSubScanSpec(scanSpec), subScan.getColumns()));
            } else {
                readers.add(new MaprDBJsonRecordReader(scanSpec, subScan.getFormatPluginConfig(), subScan.getColumns(), context));
            }
        } catch (Exception e) {
            throw new ExecutionSetupException(e);
        }
    }
    return new ScanBatch(subScan, context, readers);
}

Example 78 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class ImplCreator method getRecordBatch.

/**
 * Create a RecordBatch and its children for given PhysicalOperator
 */
@VisibleForTesting
public RecordBatch getRecordBatch(final PhysicalOperator op, final ExecutorFragmentContext context) throws ExecutionSetupException {
    Preconditions.checkNotNull(op);
    final List<RecordBatch> childRecordBatches = getChildren(op, context);
    if (context.isImpersonationEnabled()) {
        final UserGroupInformation proxyUgi = ImpersonationUtil.createProxyUgi(op.getUserName(), context.getQueryUserName());
        try {
            return proxyUgi.doAs(new PrivilegedExceptionAction<RecordBatch>() {

                @Override
                public RecordBatch run() throws Exception {
                    @SuppressWarnings("unchecked") final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(context, op, childRecordBatches);
                    operators.addFirst(batch);
                    return batch;
                }
            });
        } catch (InterruptedException | IOException e) {
            final String errMsg = String.format("Failed to create RecordBatch for operator with id '%d'", op.getOperatorId());
            logger.error(errMsg, e);
            throw new ExecutionSetupException(errMsg, e);
        }
    } else {
        @SuppressWarnings("unchecked") final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(context, op, childRecordBatches);
        operators.addFirst(batch);
        return batch;
    }
}

Example 79 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class DrillParquetReader method setup.

@Override
public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
    try {
        this.operatorContext = context;
        schema = footer.getFileMetaData().getSchema();
        MessageType projection;
        if (isStarQuery()) {
            projection = schema;
        } else {
            columnsNotFound = new ArrayList<>();
            projection = getProjection(schema, getColumns(), columnsNotFound);
            if (projection == null) {
                projection = schema;
            }
            if (columnsNotFound != null && columnsNotFound.size() > 0) {
                nullFilledVectors = new ArrayList<>();
                for (SchemaPath col : columnsNotFound) {
                    // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps
                    nullFilledVectors.add((NullableIntVector) output.addField(MaterializedField.create(col.toExpr(), org.apache.drill.common.types.Types.optional(TypeProtos.MinorType.INT)), (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT, TypeProtos.DataMode.OPTIONAL)));
                }
                if (columnsNotFound.size() == getColumns().size()) {
                    noColumnsFound = true;
                }
            }
        }
        logger.debug("Requesting schema {}", projection);
        ColumnIOFactory factory = new ColumnIOFactory(false);
        MessageColumnIO columnIO = factory.getColumnIO(projection, schema);
        Map<ColumnPath, ColumnChunkMetaData> paths = new HashMap<>();
        for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
            paths.put(md.getPath(), md);
        }
        Path filePath = new Path(entry.getPath());
        BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());
        recordCount = (int) blockMetaData.getRowCount();
        pageReadStore = new ColumnChunkIncReadStore(recordCount, CodecFactory.createDirectCodecFactory(fileSystem.getConf(), new ParquetDirectByteBufferAllocator(operatorContext.getAllocator()), 0), operatorContext.getAllocator(), fileSystem, filePath);
        for (String[] path : schema.getPaths()) {
            Type type = schema.getType(path);
            if (type.isPrimitive()) {
                ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
                pageReadStore.addColumn(schema.getColumnDescription(path), md);
            }
        }
        if (!noColumnsFound) {
            // Discard the columns not found in the schema when create DrillParquetRecordMaterializer, since they have been added to output already.
            @SuppressWarnings("unchecked") final Collection<SchemaPath> columns = columnsNotFound == null || columnsNotFound.size() == 0 ? getColumns() : CollectionUtils.subtract(getColumns(), columnsNotFound);
            recordMaterializer = new DrillParquetRecordMaterializer(output, projection, columns, fragmentContext.getOptions(), containsCorruptedDates);
            recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);
        }
    } catch (Exception e) {
        handleAndRaise("Failure in setting up reader", e);
    }
}

Example 80 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.

the class LegacyOperatorTestBuilder method go.

@SuppressWarnings("unchecked")
public void go() {
    BatchCreator<PhysicalOperator> opCreator;
    RecordBatch testOperator;
    try {
        physicalOpUnitTestBase.mockOpContext(popConfig, initReservation, maxAllocation);
        opCreator = (BatchCreator<PhysicalOperator>) physicalOpUnitTestBase.opCreatorReg.getOperatorCreator(popConfig.getClass());
        List<RecordBatch> incomingStreams = Lists.newArrayList();
        if (inputStreamsJSON != null) {
            for (List<String> batchesJson : inputStreamsJSON) {
                incomingStreams.add(new ScanBatch(popConfig, physicalOpUnitTestBase.fragContext, physicalOpUnitTestBase.getReaderListForJsonBatches(batchesJson, physicalOpUnitTestBase.fragContext)));
            }
        }
        testOperator = opCreator.getBatch(physicalOpUnitTestBase.fragContext, popConfig, incomingStreams);
        Map<String, List<Object>> actualSuperVectors = DrillTestWrapper.addToCombinedVectorResults(new PhysicalOpUnitTestBase.BatchIterator(testOperator), expectedBatchSize, expectedNumBatches, expectedTotalRows);
        // when checking total rows, don't compare actual results
        if (expectedTotalRows != null) {
            return;
        }
        Map<String, List<Object>> expectedSuperVectors;
        if (expectNoRows) {
            expectedSuperVectors = new TreeMap<>();
            for (String column : baselineColumns) {
                expectedSuperVectors.put(column, new ArrayList<>());
            }
        } else {
            expectedSuperVectors = DrillTestWrapper.translateRecordListToHeapVectors(baselineRecords);
        }
        DrillTestWrapper.compareMergedVectors(expectedSuperVectors, actualSuperVectors);
    } catch (ExecutionSetupException e) {
        throw new RuntimeException(e);
    } catch (UnsupportedEncodingException e) {
        throw new RuntimeException(e);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}