Examples with Field com.twitter.elephantbird.thrift.TStructDescriptor.Field used on opensource projects

Example 6 with Field

use of com.twitter.elephantbird.thrift.TStructDescriptor.Field in project elephant-bird by twitter.

the class ThriftToPig method toPigMap.

private static Map<String, Object> toPigMap(Field field, Map<Object, Object> map, boolean lazy) {
    // PIG map's key always a String. just use toString() and hope
    // things would work out ok.
    HashMap<String, Object> out = new HashMap<String, Object>(map.size());
    Field valueField = field.getMapValueField();
    for (Entry<Object, Object> e : map.entrySet()) {
        String key = e.getKey() == null ? null : e.getKey().toString();
        Object prev = out.put(key, toPigObject(valueField, e.getValue(), lazy));
        if (prev != null) {
            String msg = "Duplicate keys while converting to String while " + " processing map " + field.getName() + " (key type : " + field.getMapKeyField().getType() + " value type : " + field.getMapValueField().getType() + ")";
            LOG.warn(msg);
            throw new RuntimeException(msg);
        }
    }
    return out;
}

Example 7 with Field

use of com.twitter.elephantbird.thrift.TStructDescriptor.Field in project parquet-mr by apache.

the class ThriftSchemaConverter method toStructType.

private static StructType toStructType(TStructDescriptor struct) {
    List<Field> fields = struct.getFields();
    List<ThriftField> children = new ArrayList<ThriftField>(fields.size());
    for (Field field : fields) {
        Requirement req = field.getFieldMetaData() == null ? Requirement.OPTIONAL : Requirement.fromType(field.getFieldMetaData().requirementType);
        children.add(toThriftField(field.getName(), field, req));
    }
    return new StructType(children, structOrUnionType(struct.getThriftClass()));
}

Example 8 with Field

use of com.twitter.elephantbird.thrift.TStructDescriptor.Field in project parquet-mr by apache.

the class ThriftSchemaConverter method toThriftField.

private static ThriftField toThriftField(String name, Field field, ThriftField.Requirement requirement) {
    ThriftType type;
    switch(ThriftTypeID.fromByte(field.getType())) {
        case STOP:
        case VOID:
        default:
            throw new UnsupportedOperationException("can't convert type of " + field);
        case BOOL:
            type = new BoolType();
            break;
        case BYTE:
            type = new ByteType();
            break;
        case DOUBLE:
            type = new DoubleType();
            break;
        case I16:
            type = new I16Type();
            break;
        case I32:
            type = new I32Type();
            break;
        case I64:
            type = new I64Type();
            break;
        case STRING:
            StringType stringType = new StringType();
            FieldMetaData fieldMetaData = field.getFieldMetaData();
            // binary data is represented by String type with an additional binary flag.
            if (fieldMetaData != null && fieldMetaData.valueMetaData.isBinary()) {
                stringType.setBinary(true);
            }
            type = stringType;
            break;
        case STRUCT:
            type = toStructType(field.gettStructDescriptor());
            break;
        case MAP:
            final Field mapKeyField = field.getMapKeyField();
            final Field mapValueField = field.getMapValueField();
            type = new ThriftType.MapType(toThriftField(mapKeyField.getName(), mapKeyField, requirement), toThriftField(mapValueField.getName(), mapValueField, requirement));
            break;
        case SET:
            final Field setElemField = field.getSetElemField();
            type = new ThriftType.SetType(toThriftField(setElemField.getName(), setElemField, requirement));
            break;
        case LIST:
            final Field listElemField = field.getListElemField();
            type = new ThriftType.ListType(toThriftField(listElemField.getName(), listElemField, requirement));
            break;
        case ENUM:
            Collection<TEnum> enumValues = field.getEnumValues();
            List<EnumValue> values = new ArrayList<ThriftType.EnumValue>();
            for (TEnum tEnum : enumValues) {
                values.add(new EnumValue(tEnum.getValue(), tEnum.toString()));
            }
            type = new EnumType(values);
            break;
    }
    return new ThriftField(name, field.getId(), requirement, type);
}

Example 9 with Field

use of com.twitter.elephantbird.thrift.TStructDescriptor.Field in project elephant-bird by twitter.

the class PigToThrift method toThriftMap.

private static Map<Object, Object> toThriftMap(Field field, Map<String, Object> map) {
    Field keyField = field.getMapKeyField();
    Field valueField = field.getMapValueField();
    HashMap<Object, Object> out = new HashMap<Object, Object>(map.size());
    for (Entry<String, Object> e : map.entrySet()) {
        String s = e.getKey();
        Object key;
        switch(keyField.getType()) {
            case TType.STRING:
                key = s;
                break;
            case TType.BOOL:
                key = Boolean.parseBoolean(s);
                break;
            case TType.BYTE:
                key = Byte.parseByte(s);
                break;
            case TType.I16:
                key = Short.parseShort(s);
                break;
            case TType.I32:
                key = Integer.parseInt(s);
                break;
            case TType.I64:
                key = Long.parseLong(s);
                break;
            case TType.DOUBLE:
                key = Double.parseDouble(s);
                break;
            case TType.ENUM:
                key = toThriftEnum(keyField, s);
                break;
            default:
                // LIST, MAP, SET, STOP, STRUCT, VOID types are unsupported
                throw new RuntimeException(String.format("Conversion from string map key to type '%s' is unsupported", ThriftUtils.getFieldValueType(keyField).getName()));
        }
        if (keyField.isBuffer()) {
            key = ByteBuffer.wrap(s.getBytes(Charsets.UTF_8));
        }
        out.put(key, toThriftValue(valueField, e.getValue()));
    }
    return out;
}

Example 10 with Field

use of com.twitter.elephantbird.thrift.TStructDescriptor.Field in project elephant-bird by twitter.

the class RCFileThriftOutputFormat method makeColumnarMetadata.

protected ColumnarMetadata makeColumnarMetadata() {
    List<Integer> fieldIds = Lists.newArrayList();
    for (Field fd : tDesc.getFields()) {
        fieldIds.add((int) fd.getFieldId());
    }
    // -1 for unknown fields
    fieldIds.add(-1);
    return ColumnarMetadata.newInstance(typeRef.getRawClass().getName(), fieldIds);
}