Examples with FrontendException org.apache.pig.impl.logicalLayer.FrontendException used on opensource projects

Example 26 with FrontendException

use of org.apache.pig.impl.logicalLayer.FrontendException in project hive by apache.

the class HCatBaseStorer method validateSchema.

/**
 * This method encodes which Pig type can map (be stored in) to which HCat type.
 * @throws HCatException
 * @throws FrontendException
 */
private void validateSchema(FieldSchema pigField, HCatFieldSchema hcatField, Schema topLevelPigSchema, HCatSchema topLevelHCatSchema, int columnPos) throws HCatException, FrontendException {
    validateAlias(pigField.alias);
    byte type = pigField.type;
    if (DataType.isComplex(type)) {
        switch(type) {
            case DataType.MAP:
                if (hcatField != null) {
                    if (hcatField.getMapKeyType() != Type.STRING) {
                        throw new FrontendException("Key Type of map must be String " + hcatField, PigHCatUtil.PIG_EXCEPTION_CODE);
                    }
                // Map values can be primitive or complex
                }
                break;
            case DataType.BAG:
                HCatSchema arrayElementSchema = hcatField == null ? null : hcatField.getArrayElementSchema();
                for (FieldSchema innerField : pigField.schema.getField(0).schema.getFields()) {
                    validateSchema(innerField, getColFromSchema(pigField.alias, arrayElementSchema), topLevelPigSchema, topLevelHCatSchema, columnPos);
                }
                break;
            case DataType.TUPLE:
                HCatSchema structSubSchema = hcatField == null ? null : hcatField.getStructSubSchema();
                for (FieldSchema innerField : pigField.schema.getFields()) {
                    validateSchema(innerField, getColFromSchema(pigField.alias, structSubSchema), topLevelPigSchema, topLevelHCatSchema, columnPos);
                }
                break;
            default:
                throw new FrontendException("Internal Error.", PigHCatUtil.PIG_EXCEPTION_CODE);
        }
    } else if (hcatField != null) {
        // there is no point trying to validate further if we have no type info about target field
        switch(type) {
            case DataType.BIGDECIMAL:
                throwTypeMismatchException(type, Lists.newArrayList(Type.DECIMAL), hcatField, columnPos);
                break;
            case DataType.DATETIME:
                throwTypeMismatchException(type, Lists.newArrayList(Type.TIMESTAMP, Type.DATE), hcatField, columnPos);
                break;
            case DataType.BYTEARRAY:
                throwTypeMismatchException(type, Lists.newArrayList(Type.BINARY), hcatField, columnPos);
                break;
            case DataType.BIGINTEGER:
                throwTypeMismatchException(type, Collections.<Type>emptyList(), hcatField, columnPos);
                break;
            case DataType.BOOLEAN:
                throwTypeMismatchException(type, Lists.newArrayList(Type.BOOLEAN), hcatField, columnPos);
                break;
            case DataType.CHARARRAY:
                throwTypeMismatchException(type, Lists.newArrayList(Type.STRING, Type.CHAR, Type.VARCHAR), hcatField, columnPos);
                break;
            case DataType.DOUBLE:
                throwTypeMismatchException(type, Lists.newArrayList(Type.DOUBLE), hcatField, columnPos);
                break;
            case DataType.FLOAT:
                throwTypeMismatchException(type, Lists.newArrayList(Type.FLOAT), hcatField, columnPos);
                break;
            case DataType.INTEGER:
                throwTypeMismatchException(type, Lists.newArrayList(Type.INT, Type.BIGINT, Type.TINYINT, Type.SMALLINT), hcatField, columnPos);
                break;
            case DataType.LONG:
                throwTypeMismatchException(type, Lists.newArrayList(Type.BIGINT), hcatField, columnPos);
                break;
            default:
                throw new FrontendException("'" + type + "' Pig datatype in column " + columnPos + "(0-based) is not supported by HCat", PigHCatUtil.PIG_EXCEPTION_CODE);
        }
    } else {
        if (false) {
            // see HIVE-6194
            throw new FrontendException("(pigSch,hcatSchema)=(" + pigField + "," + "" + hcatField + ") (topPig, topHcat)=(" + topLevelPigSchema + "," + "" + topLevelHCatSchema + ")");
        }
    }
}

Example 27 with FrontendException

use of org.apache.pig.impl.logicalLayer.FrontendException in project hive by apache.

the class HCatBaseStorer method checkSchema.

@Override
public void checkSchema(ResourceSchema resourceSchema) throws IOException {
    /*  Schema provided by user and the schema computed by Pig
    * at the time of calling store must match.
    */
    Schema runtimeSchema = Schema.getPigSchema(resourceSchema);
    if (pigSchema != null) {
        if (!Schema.equals(runtimeSchema, pigSchema, false, true)) {
            throw new FrontendException("Schema provided in store statement doesn't match with the Schema" + "returned by Pig run-time. Schema provided in HCatStorer: " + pigSchema.toString() + " Schema received from Pig runtime: " + runtimeSchema.toString(), PigHCatUtil.PIG_EXCEPTION_CODE);
        }
    } else {
        pigSchema = runtimeSchema;
    }
    UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] { sign }).setProperty(PIG_SCHEMA, ObjectSerializer.serialize(pigSchema));
}

Example 28 with FrontendException

use of org.apache.pig.impl.logicalLayer.FrontendException in project hive by apache.

the class HCatStorer method setStoreLocation.

/**
 * @param location databaseName.tableName
 */
@Override
public void setStoreLocation(String location, Job job) throws IOException {
    Configuration config = job.getConfiguration();
    config.set(INNER_SIGNATURE, INNER_SIGNATURE_PREFIX + "_" + sign);
    Properties udfProps = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] { sign });
    String[] userStr = location.split("\\.");
    if (udfProps.containsKey(HCatConstants.HCAT_PIG_STORER_LOCATION_SET)) {
        for (Enumeration<Object> emr = udfProps.keys(); emr.hasMoreElements(); ) {
            PigHCatUtil.getConfigFromUDFProperties(udfProps, config, emr.nextElement().toString());
        }
        Credentials crd = jobCredentials.get(INNER_SIGNATURE_PREFIX + "_" + sign);
        if (crd != null) {
            job.getCredentials().addAll(crd);
        }
    } else {
        Job clone = new Job(job.getConfiguration());
        OutputJobInfo outputJobInfo;
        if (userStr.length == 2) {
            outputJobInfo = OutputJobInfo.create(userStr[0], userStr[1], partitions);
        } else if (userStr.length == 1) {
            outputJobInfo = OutputJobInfo.create(null, userStr[0], partitions);
        } else {
            throw new FrontendException("location " + location + " is invalid. It must be of the form [db.]table", PigHCatUtil.PIG_EXCEPTION_CODE);
        }
        Schema schema = (Schema) ObjectSerializer.deserialize(udfProps.getProperty(PIG_SCHEMA));
        if (schema != null) {
            pigSchema = schema;
        }
        if (pigSchema == null) {
            throw new FrontendException("Schema for data cannot be determined.", PigHCatUtil.PIG_EXCEPTION_CODE);
        }
        String externalLocation = (String) udfProps.getProperty(HCatConstants.HCAT_PIG_STORER_EXTERNAL_LOCATION);
        if (externalLocation != null) {
            outputJobInfo.setLocation(externalLocation);
        }
        try {
            HCatOutputFormat.setOutput(job, outputJobInfo);
        } catch (HCatException he) {
            // information passed to HCatOutputFormat was not right
            throw new PigException(he.getMessage(), PigHCatUtil.PIG_EXCEPTION_CODE, he);
        }
        HCatSchema hcatTblSchema = HCatOutputFormat.getTableSchema(job.getConfiguration());
        try {
            doSchemaValidations(pigSchema, hcatTblSchema);
        } catch (HCatException he) {
            throw new FrontendException(he.getMessage(), PigHCatUtil.PIG_EXCEPTION_CODE, he);
        }
        computedSchema = convertPigSchemaToHCatSchema(pigSchema, hcatTblSchema);
        HCatOutputFormat.setSchema(job, computedSchema);
        udfProps.setProperty(COMPUTED_OUTPUT_SCHEMA, ObjectSerializer.serialize(computedSchema));
        // methods need not be called many times.
        for (Entry<String, String> keyValue : job.getConfiguration()) {
            String oldValue = clone.getConfiguration().getRaw(keyValue.getKey());
            if ((oldValue == null) || (keyValue.getValue().equals(oldValue) == false)) {
                udfProps.put(keyValue.getKey(), keyValue.getValue());
            }
        }
        // Store credentials in a private hash map and not the udf context to
        // make sure they are not public.
        jobCredentials.put(INNER_SIGNATURE_PREFIX + "_" + sign, job.getCredentials());
        udfProps.put(HCatConstants.HCAT_PIG_STORER_LOCATION_SET, true);
    }
}

Example 29 with FrontendException

use of org.apache.pig.impl.logicalLayer.FrontendException in project metacat by Netflix.

the class PigTypeConverter method fromMetacatType.

/**
 * {@inheritDoc}.
 */
@Override
public String fromMetacatType(@Nonnull @NonNull final Type type) {
    final Schema schema = new Schema(Util.translateFieldSchema(fromCanonicalTypeToPigSchema(null, type)));
    final StringBuilder result = new StringBuilder();
    try {
        Schema.stringifySchema(result, schema, DataType.GENERIC_WRITABLECOMPARABLE, Integer.MIN_VALUE);
    } catch (FrontendException e) {
        throw new IllegalArgumentException(String.format("Invalid for Pig converter: '%s'", type.getDisplayName()));
    }
    return result.toString();
}

Example 30 with FrontendException

use of org.apache.pig.impl.logicalLayer.FrontendException in project sketches-pig by DataSketches.

the class ReservoirUnion method outputSchema.

/**
 * Validates format of input schema and returns a matching schema
 * @param input Expects input to be a bag of sketches: <tt>(n, k, {(samples...)})</tt>
 * @return Schema based on the
 */
@Override
public Schema outputSchema(final Schema input) {
    if (input != null && input.size() > 0) {
        try {
            Schema source = input;
            // if we have a bag, grab one level down to get a tuple
            if (source.size() == 1 && source.getField(0).type == DataType.BAG) {
                source = source.getField(0).schema;
            }
            if (source.size() == 1 && source.getField(0).type == DataType.TUPLE) {
                source = source.getField(0).schema;
            }
            final List<Schema.FieldSchema> fields = source.getFields();
            if (fields.size() == 3 && fields.get(0).type == DataType.LONG && fields.get(1).type == DataType.INTEGER && fields.get(2).type == DataType.BAG) {
                return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), source), source, DataType.TUPLE));
            }
        } catch (final FrontendException e) {
            throw new RuntimeException(e);
        }
    }
    return null;
}