Examples with ExecutionSetupException org.apache.drill.common.exceptions.ExecutionSetupException used on opensource projects

Example 61 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by apache.

the class ImplCreator method getRecordBatch.

/** Create a RecordBatch and its children for given PhysicalOperator */
@VisibleForTesting
public RecordBatch getRecordBatch(final PhysicalOperator op, final FragmentContext context) throws ExecutionSetupException {
    Preconditions.checkNotNull(op);
    final List<RecordBatch> childRecordBatches = getChildren(op, context);
    if (context.isImpersonationEnabled()) {
        final UserGroupInformation proxyUgi = ImpersonationUtil.createProxyUgi(op.getUserName(), context.getQueryUserName());
        try {
            return proxyUgi.doAs(new PrivilegedExceptionAction<RecordBatch>() {

                @Override
                public RecordBatch run() throws Exception {
                    final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(context, op, childRecordBatches);
                    operators.addFirst(batch);
                    return batch;
                }
            });
        } catch (InterruptedException | IOException e) {
            final String errMsg = String.format("Failed to create RecordBatch for operator with id '%d'", op.getOperatorId());
            logger.error(errMsg, e);
            throw new ExecutionSetupException(errMsg, e);
        }
    } else {
        final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(context, op, childRecordBatches);
        operators.addFirst(batch);
        return batch;
    }
}

Example 62 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class HiveStoragePlugin method registerSchemas.

// Forced to synchronize this method to allow error recovery
// in the multi-threaded case. Can remove synchronized only
// by restructuring connections and cache to allow better
// recovery from failed secure connections.
@Override
public synchronized void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException {
    try {
        schemaFactory.registerSchemas(schemaConfig, parent);
        return;
    // Hack. We may need to retry the connection. But, we can't because
    // the retry logic is implemented in the very connection we need to
    // discard and rebuild. To work around, we discard the entire schema
    // factory, and all its invalid connections. Very crude, but the
    // easiest short-term solution until we refactor the code to do the
    // job properly. See DRILL-5510.
    } catch (Throwable e) {
        // Unwrap exception
        Throwable ex = e;
        for (; ; ) {
            // Case for failing on an invalid cached connection
            if (ex instanceof MetaException || // tokens.
            ex instanceof TTransportException) {
                break;
            }
            if (ex.getCause() == null || ex.getCause() == ex) {
                logger.error("Hive metastore register schemas failed", e);
                throw new DrillRuntimeException("Unknown Hive error", e);
            }
            ex = ex.getCause();
        }
    }
    try {
        schemaFactory.close();
    } catch (Throwable t) {
        // Ignore, we're in a bad state.
        logger.warn("Schema factory forced close failed, error ignored", t);
    }
    try {
        schemaFactory = new HiveSchemaFactory(this, name, hiveConf);
    } catch (ExecutionSetupException e) {
        throw new DrillRuntimeException(e);
    }
    // Try the schemas again. If this fails, just give up.
    schemaFactory.registerSchemas(schemaConfig, parent);
    logger.debug("Successfully recovered from a Hive metastore connection failure.");
}

Example 63 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class HiveUtilities method getInputFormatClass.

/**
 * Utility method which gets table or partition {@link InputFormat} class. First it
 * tries to get the class name from given StorageDescriptor object. If it doesn't contain it tries to get it from
 * StorageHandler class set in table properties. If not found throws an exception.
 * @param job {@link JobConf} instance needed incase the table is StorageHandler based table.
 * @param sd {@link StorageDescriptor} instance of currently reading partition or table (for non-partitioned tables).
 * @param table Table object
 * @throws Exception
 */
public static Class<? extends InputFormat<?, ?>> getInputFormatClass(final JobConf job, final StorageDescriptor sd, final Table table) throws Exception {
    final String inputFormatName = sd.getInputFormat();
    if (Strings.isNullOrEmpty(inputFormatName)) {
        final String storageHandlerClass = table.getParameters().get(META_TABLE_STORAGE);
        if (Strings.isNullOrEmpty(storageHandlerClass)) {
            throw new ExecutionSetupException("Unable to get Hive table InputFormat class. There is neither " + "InputFormat class explicitly specified nor StorageHandler class");
        }
        final HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(job, storageHandlerClass);
        TableDesc tableDesc = new TableDesc();
        tableDesc.setProperties(MetaStoreUtils.getTableMetadata(table));
        storageHandler.configureInputJobProperties(tableDesc, table.getParameters());
        return (Class<? extends InputFormat<?, ?>>) storageHandler.getInputFormatClass();
    } else {
        return (Class<? extends InputFormat<?, ?>>) Class.forName(inputFormatName);
    }
}

Example 64 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class HiveAbstractReader method init.

private void init() throws ExecutionSetupException {
    job = new JobConf(hiveConf);
    // Get the configured default val
    defaultPartitionValue = hiveConf.get(ConfVars.DEFAULTPARTITIONNAME.varname);
    Properties tableProperties;
    try {
        tableProperties = HiveUtilities.getTableMetadata(table);
        final Properties partitionProperties = (partition == null) ? tableProperties : HiveUtilities.getPartitionMetadata(partition, table);
        HiveUtilities.addConfToJob(job, partitionProperties);
        final Deserializer tableDeserializer = createDeserializer(job, table.getSd().getSerdeInfo().getSerializationLib(), tableProperties);
        final StructObjectInspector tableOI = getStructOI(tableDeserializer);
        if (partition != null) {
            partitionDeserializer = createDeserializer(job, partition.getSd().getSerdeInfo().getSerializationLib(), partitionProperties);
            partitionOI = getStructOI(partitionDeserializer);
            finalOI = (StructObjectInspector) ObjectInspectorConverters.getConvertedOI(partitionOI, tableOI);
            partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(partitionOI, finalOI);
            job.setInputFormat(HiveUtilities.getInputFormatClass(job, partition.getSd(), table));
            HiveUtilities.verifyAndAddTransactionalProperties(job, partition.getSd());
        } else {
            // For non-partitioned tables, there is no need to create converter as there are no schema changes expected.
            partitionDeserializer = tableDeserializer;
            partitionOI = tableOI;
            partTblObjectInspectorConverter = null;
            finalOI = tableOI;
            job.setInputFormat(HiveUtilities.getInputFormatClass(job, table.getSd(), table));
            HiveUtilities.verifyAndAddTransactionalProperties(job, table.getSd());
        }
        if (logger.isTraceEnabled()) {
            for (StructField field : finalOI.getAllStructFieldRefs()) {
                logger.trace("field in finalOI: {}", field.getClass().getName());
            }
            logger.trace("partitionDeserializer class is {} {}", partitionDeserializer.getClass().getName());
        }
        // Get list of partition column names
        final List<String> partitionNames = Lists.newArrayList();
        for (FieldSchema field : table.getPartitionKeys()) {
            partitionNames.add(field.getName());
        }
        // We should always get the columns names from ObjectInspector. For some of the tables (ex. avro) metastore
        // may not contain the schema, instead it is derived from other sources such as table properties or external file.
        // Deserializer object knows how to get the schema with all the config and table properties passed in initialization.
        // ObjectInspector created from the Deserializer object has the schema.
        final StructTypeInfo sTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(finalOI);
        final List<String> tableColumnNames = sTypeInfo.getAllStructFieldNames();
        // Select list of columns for project pushdown into Hive SerDe readers.
        final List<Integer> columnIds = Lists.newArrayList();
        if (isStarQuery()) {
            selectedColumnNames = tableColumnNames;
            for (int i = 0; i < selectedColumnNames.size(); i++) {
                columnIds.add(i);
            }
            selectedPartitionNames = partitionNames;
        } else {
            selectedColumnNames = Lists.newArrayList();
            for (SchemaPath field : getColumns()) {
                String columnName = field.getRootSegment().getPath();
                if (partitionNames.contains(columnName)) {
                    selectedPartitionNames.add(columnName);
                } else {
                    columnIds.add(tableColumnNames.indexOf(columnName));
                    selectedColumnNames.add(columnName);
                }
            }
        }
        ColumnProjectionUtils.appendReadColumns(job, columnIds);
        for (String columnName : selectedColumnNames) {
            StructField fieldRef = finalOI.getStructFieldRef(columnName);
            selectedStructFieldRefs.add(fieldRef);
            ObjectInspector fieldOI = fieldRef.getFieldObjectInspector();
            TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fieldOI.getTypeName());
            selectedColumnObjInspectors.add(fieldOI);
            selectedColumnTypes.add(typeInfo);
            selectedColumnFieldConverters.add(HiveFieldConverter.create(typeInfo, fragmentContext));
        }
        for (int i = 0; i < selectedColumnNames.size(); ++i) {
            logger.trace("inspector:typeName={}, className={}, TypeInfo: {}, converter:{}", selectedColumnObjInspectors.get(i).getTypeName(), selectedColumnObjInspectors.get(i).getClass().getName(), selectedColumnTypes.get(i).toString(), selectedColumnFieldConverters.get(i).getClass().getName());
        }
        for (int i = 0; i < table.getPartitionKeys().size(); i++) {
            FieldSchema field = table.getPartitionKeys().get(i);
            if (selectedPartitionNames.contains(field.getName())) {
                TypeInfo pType = TypeInfoUtils.getTypeInfoFromTypeString(field.getType());
                selectedPartitionTypes.add(pType);
                if (partition != null) {
                    selectedPartitionValues.add(HiveUtilities.convertPartitionType(pType, partition.getValues().get(i), defaultPartitionValue));
                }
            }
        }
    } catch (Exception e) {
        throw new ExecutionSetupException("Failure while initializing Hive Reader " + this.getClass().getName(), e);
    }
    if (!empty && initNextReader(job)) {
        internalInit(tableProperties, reader);
    }
}

Example 65 with ExecutionSetupException

use of org.apache.drill.common.exceptions.ExecutionSetupException in project drill by axbaretto.

the class StoragePluginRegistryImpl method create.

private StoragePlugin create(String name, StoragePluginConfig pluginConfig) throws ExecutionSetupException {
    StoragePlugin plugin = null;
    Constructor<? extends StoragePlugin> c = availablePlugins.get(pluginConfig.getClass());
    if (c == null) {
        throw new ExecutionSetupException(String.format("Failure finding StoragePlugin constructor for config %s", pluginConfig));
    }
    try {
        plugin = c.newInstance(pluginConfig, context, name);
        plugin.start();
        return plugin;
    } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | IOException e) {
        Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException) e).getTargetException() : e;
        if (t instanceof ExecutionSetupException) {
            throw ((ExecutionSetupException) t);
        }
        throw new ExecutionSetupException(String.format("Failure setting up new storage plugin configuration for config %s", pluginConfig), t);
    }
}