Examples with HiveCharWritable org.apache.hadoop.hive.serde2.io.HiveCharWritable used on opensource projects

Example 21 with HiveCharWritable

use of org.apache.hadoop.hive.serde2.io.HiveCharWritable in project hive by apache.

the class BinarySortableSerDe method serialize.

static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, boolean invert, byte nullMarker, byte notNullMarker) throws SerDeException {
    // Is this field a null?
    if (o == null) {
        writeByte(buffer, nullMarker, invert);
        return;
    }
    // This field is not a null.
    writeByte(buffer, notNullMarker, invert);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return;
                        }
                    case BOOLEAN:
                        {
                            boolean v = ((BooleanObjectInspector) poi).get(o);
                            writeByte(buffer, (byte) (v ? 2 : 1), invert);
                            return;
                        }
                    case BYTE:
                        {
                            ByteObjectInspector boi = (ByteObjectInspector) poi;
                            byte v = boi.get(o);
                            writeByte(buffer, (byte) (v ^ 0x80), invert);
                            return;
                        }
                    case SHORT:
                        {
                            ShortObjectInspector spoi = (ShortObjectInspector) poi;
                            short v = spoi.get(o);
                            serializeShort(buffer, v, invert);
                            return;
                        }
                    case INT:
                        {
                            IntObjectInspector ioi = (IntObjectInspector) poi;
                            int v = ioi.get(o);
                            serializeInt(buffer, v, invert);
                            return;
                        }
                    case LONG:
                        {
                            LongObjectInspector loi = (LongObjectInspector) poi;
                            long v = loi.get(o);
                            serializeLong(buffer, v, invert);
                            return;
                        }
                    case FLOAT:
                        {
                            FloatObjectInspector foi = (FloatObjectInspector) poi;
                            serializeFloat(buffer, foi.get(o), invert);
                            return;
                        }
                    case DOUBLE:
                        {
                            DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                            serializeDouble(buffer, doi.get(o), invert);
                            return;
                        }
                    case STRING:
                        {
                            StringObjectInspector soi = (StringObjectInspector) poi;
                            Text t = soi.getPrimitiveWritableObject(o);
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case CHAR:
                        {
                            HiveCharObjectInspector hcoi = (HiveCharObjectInspector) poi;
                            HiveCharWritable hc = hcoi.getPrimitiveWritableObject(o);
                            // Trailing space should ignored for char comparisons.
                            // So write stripped values for this SerDe.
                            Text t = hc.getStrippedValue();
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case VARCHAR:
                        {
                            HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) poi;
                            HiveVarcharWritable hc = hcoi.getPrimitiveWritableObject(o);
                            // use varchar's text field directly
                            Text t = hc.getTextValue();
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case BINARY:
                        {
                            BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                            BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                            byte[] toSer = new byte[ba.getLength()];
                            System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                            serializeBytes(buffer, toSer, ba.getLength(), invert);
                            return;
                        }
                    case DATE:
                        {
                            DateObjectInspector doi = (DateObjectInspector) poi;
                            int v = doi.getPrimitiveWritableObject(o).getDays();
                            serializeInt(buffer, v, invert);
                            return;
                        }
                    case TIMESTAMP:
                        {
                            TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                            TimestampWritable t = toi.getPrimitiveWritableObject(o);
                            serializeTimestampWritable(buffer, t, invert);
                            return;
                        }
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonthObjectInspector ioi = (HiveIntervalYearMonthObjectInspector) poi;
                            HiveIntervalYearMonth intervalYearMonth = ioi.getPrimitiveJavaObject(o);
                            serializeHiveIntervalYearMonth(buffer, intervalYearMonth, invert);
                            return;
                        }
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTimeObjectInspector ioi = (HiveIntervalDayTimeObjectInspector) poi;
                            HiveIntervalDayTime intervalDayTime = ioi.getPrimitiveJavaObject(o);
                            serializeHiveIntervalDayTime(buffer, intervalDayTime, invert);
                            return;
                        }
                    case DECIMAL:
                        {
                            HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi;
                            HiveDecimal dec = boi.getPrimitiveJavaObject(o);
                            serializeHiveDecimal(buffer, dec, invert);
                            return;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector eoi = loi.getListElementObjectInspector();
                // \1 followed by each element
                int size = loi.getListLength(o);
                for (int eid = 0; eid < size; eid++) {
                    writeByte(buffer, (byte) 1, invert);
                    serialize(buffer, loi.getListElement(o, eid), eoi, invert, nullMarker, notNullMarker);
                }
                // and \0 to terminate
                writeByte(buffer, (byte) 0, invert);
                return;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector koi = moi.getMapKeyObjectInspector();
                ObjectInspector voi = moi.getMapValueObjectInspector();
                // \1 followed by each key and then each value
                Map<?, ?> map = moi.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    writeByte(buffer, (byte) 1, invert);
                    serialize(buffer, entry.getKey(), koi, invert, nullMarker, notNullMarker);
                    serialize(buffer, entry.getValue(), voi, invert, nullMarker, notNullMarker);
                }
                // and \0 to terminate
                writeByte(buffer, (byte) 0, invert);
                return;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> fields = soi.getAllStructFieldRefs();
                for (int i = 0; i < fields.size(); i++) {
                    serialize(buffer, soi.getStructFieldData(o, fields.get(i)), fields.get(i).getFieldObjectInspector(), invert, nullMarker, notNullMarker);
                }
                return;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                byte tag = uoi.getTag(o);
                writeByte(buffer, tag, invert);
                serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert, nullMarker, notNullMarker);
                return;
            }
        default:
            {
                throw new RuntimeException("Unrecognized type: " + oi.getCategory());
            }
    }
}

Example 22 with HiveCharWritable

use of org.apache.hadoop.hive.serde2.io.HiveCharWritable in project hive by apache.

the class BinarySortableSerDe method deserialize.

static Object deserialize(InputByteBuffer buffer, TypeInfo type, boolean invert, byte nullMarker, byte notNullMarker, Object reuse) throws IOException {
    // Is this field a null?
    byte isNull = buffer.read(invert);
    if (isNull == nullMarker) {
        return null;
    }
    assert (isNull == notNullMarker);
    switch(type.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type;
                switch(ptype.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return null;
                        }
                    case BOOLEAN:
                        {
                            BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
                            byte b = buffer.read(invert);
                            assert (b == 1 || b == 2);
                            r.set(b == 2);
                            return r;
                        }
                    case BYTE:
                        {
                            ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse;
                            r.set((byte) (buffer.read(invert) ^ 0x80));
                            return r;
                        }
                    case SHORT:
                        {
                            ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse;
                            int v = buffer.read(invert) ^ 0x80;
                            v = (v << 8) + (buffer.read(invert) & 0xff);
                            r.set((short) v);
                            return r;
                        }
                    case INT:
                        {
                            IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse;
                            r.set(deserializeInt(buffer, invert));
                            return r;
                        }
                    case LONG:
                        {
                            LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse;
                            r.set(deserializeLong(buffer, invert));
                            return r;
                        }
                    case FLOAT:
                        {
                            FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse;
                            int v = 0;
                            for (int i = 0; i < 4; i++) {
                                v = (v << 8) + (buffer.read(invert) & 0xff);
                            }
                            if ((v & (1 << 31)) == 0) {
                                // negative number, flip all bits
                                v = ~v;
                            } else {
                                // positive number, flip the first bit
                                v = v ^ (1 << 31);
                            }
                            r.set(Float.intBitsToFloat(v));
                            return r;
                        }
                    case DOUBLE:
                        {
                            DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse;
                            long v = 0;
                            for (int i = 0; i < 8; i++) {
                                v = (v << 8) + (buffer.read(invert) & 0xff);
                            }
                            if ((v & (1L << 63)) == 0) {
                                // negative number, flip all bits
                                v = ~v;
                            } else {
                                // positive number, flip the first bit
                                v = v ^ (1L << 63);
                            }
                            r.set(Double.longBitsToDouble(v));
                            return r;
                        }
                    case STRING:
                        {
                            Text r = reuse == null ? new Text() : (Text) reuse;
                            return deserializeText(buffer, invert, r);
                        }
                    case CHAR:
                        {
                            HiveCharWritable r = reuse == null ? new HiveCharWritable() : (HiveCharWritable) reuse;
                            // Use internal text member to read value
                            deserializeText(buffer, invert, r.getTextValue());
                            r.enforceMaxLength(getCharacterMaxLength(type));
                            return r;
                        }
                    case VARCHAR:
                        {
                            HiveVarcharWritable r = reuse == null ? new HiveVarcharWritable() : (HiveVarcharWritable) reuse;
                            // Use HiveVarchar's internal Text member to read the value.
                            deserializeText(buffer, invert, r.getTextValue());
                            // If we cache helper data for deserialization we could avoid having
                            // to call getVarcharMaxLength() on every deserialize call.
                            r.enforceMaxLength(getCharacterMaxLength(type));
                            return r;
                        }
                    case BINARY:
                        {
                            BytesWritable bw = new BytesWritable();
                            // Get the actual length first
                            int start = buffer.tell();
                            int length = 0;
                            do {
                                byte b = buffer.read(invert);
                                if (b == 0) {
                                    // end of string
                                    break;
                                }
                                if (b == 1) {
                                    // the last char is an escape char. read the actual char
                                    buffer.read(invert);
                                }
                                length++;
                            } while (true);
                            if (length == buffer.tell() - start) {
                                // No escaping happened, so we are already done.
                                bw.set(buffer.getData(), start, length);
                            } else {
                                // Escaping happened, we need to copy byte-by-byte.
                                // 1. Set the length first.
                                bw.set(buffer.getData(), start, length);
                                // 2. Reset the pointer.
                                buffer.seek(start);
                                // 3. Copy the data.
                                byte[] rdata = bw.getBytes();
                                for (int i = 0; i < length; i++) {
                                    byte b = buffer.read(invert);
                                    if (b == 1) {
                                        // The last char is an escape char, read the actual char.
                                        // The serialization format escape \0 to \1, and \1 to \2,
                                        // to make sure the string is null-terminated.
                                        b = (byte) (buffer.read(invert) - 1);
                                    }
                                    rdata[i] = b;
                                }
                                // 4. Read the null terminator.
                                byte b = buffer.read(invert);
                                assert (b == 0);
                            }
                            return bw;
                        }
                    case DATE:
                        {
                            DateWritable d = reuse == null ? new DateWritable() : (DateWritable) reuse;
                            d.set(deserializeInt(buffer, invert));
                            return d;
                        }
                    case TIMESTAMP:
                        TimestampWritable t = (reuse == null ? new TimestampWritable() : (TimestampWritable) reuse);
                        byte[] bytes = new byte[TimestampWritable.BINARY_SORTABLE_LENGTH];
                        for (int i = 0; i < bytes.length; i++) {
                            bytes[i] = buffer.read(invert);
                        }
                        t.setBinarySortable(bytes, 0);
                        return t;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonthWritable i = reuse == null ? new HiveIntervalYearMonthWritable() : (HiveIntervalYearMonthWritable) reuse;
                            i.set(deserializeInt(buffer, invert));
                            return i;
                        }
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTimeWritable i = reuse == null ? new HiveIntervalDayTimeWritable() : (HiveIntervalDayTimeWritable) reuse;
                            long totalSecs = deserializeLong(buffer, invert);
                            int nanos = deserializeInt(buffer, invert);
                            i.set(totalSecs, nanos);
                            return i;
                        }
                    case DECIMAL:
                        {
                            // See serialization of decimal for explanation (below)
                            HiveDecimalWritable bdw = (reuse == null ? new HiveDecimalWritable() : (HiveDecimalWritable) reuse);
                            int b = buffer.read(invert) - 1;
                            assert (b == 1 || b == -1 || b == 0);
                            boolean positive = b != -1;
                            int factor = buffer.read(invert) ^ 0x80;
                            for (int i = 0; i < 3; i++) {
                                factor = (factor << 8) + (buffer.read(invert) & 0xff);
                            }
                            if (!positive) {
                                factor = -factor;
                            }
                            int start = buffer.tell();
                            int length = 0;
                            do {
                                b = buffer.read(positive ? invert : !invert);
                                assert (b != 1);
                                if (b == 0) {
                                    // end of digits
                                    break;
                                }
                                length++;
                            } while (true);
                            final byte[] decimalBuffer = new byte[length];
                            buffer.seek(start);
                            for (int i = 0; i < length; ++i) {
                                decimalBuffer[i] = buffer.read(positive ? invert : !invert);
                            }
                            // read the null byte again
                            buffer.read(positive ? invert : !invert);
                            String digits = new String(decimalBuffer, 0, length, decimalCharSet);
                            BigInteger bi = new BigInteger(digits);
                            HiveDecimal bd = HiveDecimal.create(bi).scaleByPowerOfTen(factor - length);
                            if (!positive) {
                                bd = bd.negate();
                            }
                            bdw.set(bd);
                            return bdw;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                ListTypeInfo ltype = (ListTypeInfo) type;
                TypeInfo etype = ltype.getListElementTypeInfo();
                // Create the list if needed
                ArrayList<Object> r = reuse == null ? new ArrayList<Object>() : (ArrayList<Object>) reuse;
                // Read the list
                int size = 0;
                while (true) {
                    int more = buffer.read(invert);
                    if (more == 0) {
                        // \0 to terminate
                        break;
                    }
                    // \1 followed by each element
                    assert (more == 1);
                    if (size == r.size()) {
                        r.add(null);
                    }
                    r.set(size, deserialize(buffer, etype, invert, nullMarker, notNullMarker, r.get(size)));
                    size++;
                }
                // Remove additional elements if the list is reused
                while (r.size() > size) {
                    r.remove(r.size() - 1);
                }
                return r;
            }
        case MAP:
            {
                MapTypeInfo mtype = (MapTypeInfo) type;
                TypeInfo ktype = mtype.getMapKeyTypeInfo();
                TypeInfo vtype = mtype.getMapValueTypeInfo();
                // Create the map if needed
                Map<Object, Object> r;
                if (reuse == null) {
                    r = new HashMap<Object, Object>();
                } else {
                    r = (HashMap<Object, Object>) reuse;
                    r.clear();
                }
                while (true) {
                    int more = buffer.read(invert);
                    if (more == 0) {
                        // \0 to terminate
                        break;
                    }
                    // \1 followed by each key and then each value
                    assert (more == 1);
                    Object k = deserialize(buffer, ktype, invert, nullMarker, notNullMarker, null);
                    Object v = deserialize(buffer, vtype, invert, nullMarker, notNullMarker, null);
                    r.put(k, v);
                }
                return r;
            }
        case STRUCT:
            {
                StructTypeInfo stype = (StructTypeInfo) type;
                List<TypeInfo> fieldTypes = stype.getAllStructFieldTypeInfos();
                int size = fieldTypes.size();
                // Create the struct if needed
                ArrayList<Object> r = reuse == null ? new ArrayList<Object>(size) : (ArrayList<Object>) reuse;
                assert (r.size() <= size);
                // Set the size of the struct
                while (r.size() < size) {
                    r.add(null);
                }
                // Read one field by one field
                for (int eid = 0; eid < size; eid++) {
                    r.set(eid, deserialize(buffer, fieldTypes.get(eid), invert, nullMarker, notNullMarker, r.get(eid)));
                }
                return r;
            }
        case UNION:
            {
                UnionTypeInfo utype = (UnionTypeInfo) type;
                StandardUnion r = reuse == null ? new StandardUnion() : (StandardUnion) reuse;
                // Read the tag
                byte tag = buffer.read(invert);
                r.setTag(tag);
                r.setObject(deserialize(buffer, utype.getAllUnionObjectTypeInfos().get(tag), invert, nullMarker, notNullMarker, null));
                return r;
            }
        default:
            {
                throw new RuntimeException("Unrecognized type: " + type.getCategory());
            }
    }
}

Example 23 with HiveCharWritable

use of org.apache.hadoop.hive.serde2.io.HiveCharWritable in project hive by apache.

the class VectorAssignRow method assignRowColumn.

/**
   * Assign a row's column object to the ColumnVector at batchIndex in the VectorizedRowBatch.
   *
   * @param batch
   * @param batchIndex
   * @param logicalColumnIndex
   * @param object    The row column object whose type is the target data type.
   */
public void assignRowColumn(VectorizedRowBatch batch, int batchIndex, int logicalColumnIndex, Object object) {
    Category targetCategory = targetCategories[logicalColumnIndex];
    if (targetCategory == null) {
        /*
       * This is a column that we don't want (i.e. not included) -- we are done.
       */
        return;
    }
    final int projectionColumnNum = projectionColumnNums[logicalColumnIndex];
    if (object == null) {
        VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
        return;
    }
    switch(targetCategory) {
        case PRIMITIVE:
            {
                PrimitiveCategory targetPrimitiveCategory = targetPrimitiveCategories[logicalColumnIndex];
                switch(targetPrimitiveCategory) {
                    case VOID:
                        VectorizedBatchUtil.setNullColIsNullValue(batch.cols[projectionColumnNum], batchIndex);
                        return;
                    case BOOLEAN:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = (((BooleanWritable) object).get() ? 1 : 0);
                        break;
                    case BYTE:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((ByteWritable) object).get();
                        break;
                    case SHORT:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((ShortWritable) object).get();
                        break;
                    case INT:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((IntWritable) object).get();
                        break;
                    case LONG:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((LongWritable) object).get();
                        break;
                    case TIMESTAMP:
                        ((TimestampColumnVector) batch.cols[projectionColumnNum]).set(batchIndex, ((TimestampWritable) object).getTimestamp());
                        break;
                    case DATE:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((DateWritable) object).getDays();
                        break;
                    case FLOAT:
                        ((DoubleColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((FloatWritable) object).get();
                        break;
                    case DOUBLE:
                        ((DoubleColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((DoubleWritable) object).get();
                        break;
                    case BINARY:
                        {
                            BytesWritable bw = (BytesWritable) object;
                            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(batchIndex, bw.getBytes(), 0, bw.getLength());
                        }
                        break;
                    case STRING:
                        {
                            Text tw = (Text) object;
                            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(batchIndex, tw.getBytes(), 0, tw.getLength());
                        }
                        break;
                    case VARCHAR:
                        {
                            // UNDONE: Performance problem with conversion to String, then bytes...
                            // We store VARCHAR type stripped of pads.
                            HiveVarchar hiveVarchar;
                            if (object instanceof HiveVarchar) {
                                hiveVarchar = (HiveVarchar) object;
                            } else {
                                hiveVarchar = ((HiveVarcharWritable) object).getHiveVarchar();
                            }
                            // TODO: HIVE-13624 Do we need maxLength checking?
                            byte[] bytes = hiveVarchar.getValue().getBytes();
                            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(batchIndex, bytes, 0, bytes.length);
                        }
                        break;
                    case CHAR:
                        {
                            // UNDONE: Performance problem with conversion to String, then bytes...
                            // We store CHAR type stripped of pads.
                            HiveChar hiveChar;
                            if (object instanceof HiveChar) {
                                hiveChar = (HiveChar) object;
                            } else {
                                hiveChar = ((HiveCharWritable) object).getHiveChar();
                            }
                            // TODO: HIVE-13624 Do we need maxLength checking?
                            // We store CHAR in vector row batch with padding stripped.
                            byte[] bytes = hiveChar.getStrippedValue().getBytes();
                            ((BytesColumnVector) batch.cols[projectionColumnNum]).setVal(batchIndex, bytes, 0, bytes.length);
                        }
                        break;
                    case DECIMAL:
                        if (object instanceof HiveDecimal) {
                            ((DecimalColumnVector) batch.cols[projectionColumnNum]).set(batchIndex, (HiveDecimal) object);
                        } else {
                            ((DecimalColumnVector) batch.cols[projectionColumnNum]).set(batchIndex, (HiveDecimalWritable) object);
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth().getTotalMonths();
                        break;
                    case INTERVAL_DAY_TIME:
                        ((IntervalDayTimeColumnVector) batch.cols[projectionColumnNum]).set(batchIndex, ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime());
                        break;
                    default:
                        throw new RuntimeException("Primitive category " + targetPrimitiveCategory.name() + " not supported");
                }
            }
            break;
        default:
            throw new RuntimeException("Category " + targetCategory.name() + " not supported");
    }
    /*
     * We always set the null flag to false when there is a value.
     */
    batch.cols[projectionColumnNum].isNull[batchIndex] = false;
}

Example 24 with HiveCharWritable

use of org.apache.hadoop.hive.serde2.io.HiveCharWritable in project hive by apache.

the class VectorColumnAssignFactory method buildObjectAssign.

public static VectorColumnAssign buildObjectAssign(VectorizedRowBatch outputBatch, int outColIndex, PrimitiveCategory category) throws HiveException {
    VectorColumnAssign outVCA = null;
    ColumnVector destCol = outputBatch.cols[outColIndex];
    if (destCol == null) {
        switch(category) {
            case VOID:
                outVCA = new VectorLongColumnAssign() {

                    // This is a dummy assigner
                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        // This is no-op, there is no column to assign to and val is expected to be null
                        assert (val == null);
                    }
                };
                break;
            default:
                throw new HiveException("Incompatible (null) vector column and primitive category " + category);
        }
    } else if (destCol instanceof LongColumnVector) {
        switch(category) {
            case BOOLEAN:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            BooleanWritable bw = (BooleanWritable) val;
                            assignLong(bw.get() ? 1 : 0, destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case BYTE:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            ByteWritable bw = (ByteWritable) val;
                            assignLong(bw.get(), destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case SHORT:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            ShortWritable bw = (ShortWritable) val;
                            assignLong(bw.get(), destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case INT:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            IntWritable bw = (IntWritable) val;
                            assignLong(bw.get(), destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case LONG:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            LongWritable bw = (LongWritable) val;
                            assignLong(bw.get(), destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case TIMESTAMP:
                outVCA = new VectorTimestampColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            assignTimestamp((TimestampWritable) val, destIndex);
                        }
                    }
                }.init(outputBatch, (TimestampColumnVector) destCol);
                break;
            case DATE:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            DateWritable bw = (DateWritable) val;
                            assignLong(bw.getDays(), destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case INTERVAL_YEAR_MONTH:
                outVCA = new VectorLongColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            HiveIntervalYearMonthWritable bw = (HiveIntervalYearMonthWritable) val;
                            assignLong(bw.getHiveIntervalYearMonth().getTotalMonths(), destIndex);
                        }
                    }
                }.init(outputBatch, (LongColumnVector) destCol);
                break;
            case INTERVAL_DAY_TIME:
                outVCA = new VectorIntervalDayTimeColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            HiveIntervalDayTimeWritable bw = (HiveIntervalDayTimeWritable) val;
                            assignIntervalDayTime(bw.getHiveIntervalDayTime(), destIndex);
                        }
                    }
                }.init(outputBatch, (IntervalDayTimeColumnVector) destCol);
                break;
            default:
                throw new HiveException("Incompatible Long vector column and primitive category " + category);
        }
    } else if (destCol instanceof DoubleColumnVector) {
        switch(category) {
            case DOUBLE:
                outVCA = new VectorDoubleColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            DoubleWritable bw = (DoubleWritable) val;
                            assignDouble(bw.get(), destIndex);
                        }
                    }
                }.init(outputBatch, (DoubleColumnVector) destCol);
                break;
            case FLOAT:
                outVCA = new VectorDoubleColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            FloatWritable bw = (FloatWritable) val;
                            assignDouble(bw.get(), destIndex);
                        }
                    }
                }.init(outputBatch, (DoubleColumnVector) destCol);
                break;
            default:
                throw new HiveException("Incompatible Double vector column and primitive category " + category);
        }
    } else if (destCol instanceof BytesColumnVector) {
        switch(category) {
            case BINARY:
                outVCA = new VectorBytesColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            BytesWritable bw = (BytesWritable) val;
                            byte[] bytes = bw.getBytes();
                            assignBytes(bytes, 0, bw.getLength(), destIndex);
                        }
                    }
                }.init(outputBatch, (BytesColumnVector) destCol);
                break;
            case STRING:
                outVCA = new VectorBytesColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            Text bw = (Text) val;
                            byte[] bytes = bw.getBytes();
                            assignBytes(bytes, 0, bw.getLength(), destIndex);
                        }
                    }
                }.init(outputBatch, (BytesColumnVector) destCol);
                break;
            case VARCHAR:
                outVCA = new VectorBytesColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            // We store VARCHAR type stripped of pads.
                            HiveVarchar hiveVarchar;
                            if (val instanceof HiveVarchar) {
                                hiveVarchar = (HiveVarchar) val;
                            } else {
                                hiveVarchar = ((HiveVarcharWritable) val).getHiveVarchar();
                            }
                            byte[] bytes = hiveVarchar.getValue().getBytes();
                            assignBytes(bytes, 0, bytes.length, destIndex);
                        }
                    }
                }.init(outputBatch, (BytesColumnVector) destCol);
                break;
            case CHAR:
                outVCA = new VectorBytesColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            // We store CHAR type stripped of pads.
                            HiveChar hiveChar;
                            if (val instanceof HiveChar) {
                                hiveChar = (HiveChar) val;
                            } else {
                                hiveChar = ((HiveCharWritable) val).getHiveChar();
                            }
                            byte[] bytes = hiveChar.getStrippedValue().getBytes();
                            assignBytes(bytes, 0, bytes.length, destIndex);
                        }
                    }
                }.init(outputBatch, (BytesColumnVector) destCol);
                break;
            default:
                throw new HiveException("Incompatible Bytes vector column and primitive category " + category);
        }
    } else if (destCol instanceof DecimalColumnVector) {
        switch(category) {
            case DECIMAL:
                outVCA = new VectorDecimalColumnAssign() {

                    @Override
                    public void assignObjectValue(Object val, int destIndex) throws HiveException {
                        if (val == null) {
                            assignNull(destIndex);
                        } else {
                            if (val instanceof HiveDecimal) {
                                assignDecimal((HiveDecimal) val, destIndex);
                            } else {
                                assignDecimal((HiveDecimalWritable) val, destIndex);
                            }
                        }
                    }
                }.init(outputBatch, (DecimalColumnVector) destCol);
                break;
            default:
                throw new HiveException("Incompatible Decimal vector column and primitive category " + category);
        }
    } else {
        throw new HiveException("Unknown vector column type " + destCol.getClass().getName());
    }
    return outVCA;
}

Example 25 with HiveCharWritable

use of org.apache.hadoop.hive.serde2.io.HiveCharWritable in project hive by apache.

the class VectorDeserializeRow method convertRowColumn.

/**
   * Convert one row column value that is the current value in deserializeRead.
   *
   * We deserialize into a writable and then pass that writable to an instance of VectorAssignRow
   * to convert the writable to the target data type and assign it into the VectorizedRowBatch.
   *
   * @param batch
   * @param batchIndex
   * @param logicalColumnIndex
   * @throws IOException
   */
private void convertRowColumn(VectorizedRowBatch batch, int batchIndex, int logicalColumnIndex) throws IOException {
    final int projectionColumnNum = projectionColumnNums[logicalColumnIndex];
    Writable convertSourceWritable = convertSourceWritables[logicalColumnIndex];
    switch(sourceCategories[logicalColumnIndex]) {
        case PRIMITIVE:
            {
                switch(sourcePrimitiveCategories[logicalColumnIndex]) {
                    case VOID:
                        convertSourceWritable = null;
                        break;
                    case BOOLEAN:
                        ((BooleanWritable) convertSourceWritable).set(deserializeRead.currentBoolean);
                        break;
                    case BYTE:
                        ((ByteWritable) convertSourceWritable).set(deserializeRead.currentByte);
                        break;
                    case SHORT:
                        ((ShortWritable) convertSourceWritable).set(deserializeRead.currentShort);
                        break;
                    case INT:
                        ((IntWritable) convertSourceWritable).set(deserializeRead.currentInt);
                        break;
                    case LONG:
                        ((LongWritable) convertSourceWritable).set(deserializeRead.currentLong);
                        break;
                    case TIMESTAMP:
                        ((TimestampWritable) convertSourceWritable).set(deserializeRead.currentTimestampWritable);
                        break;
                    case DATE:
                        ((DateWritable) convertSourceWritable).set(deserializeRead.currentDateWritable);
                        break;
                    case FLOAT:
                        ((FloatWritable) convertSourceWritable).set(deserializeRead.currentFloat);
                        break;
                    case DOUBLE:
                        ((DoubleWritable) convertSourceWritable).set(deserializeRead.currentDouble);
                        break;
                    case BINARY:
                        if (deserializeRead.currentBytes == null) {
                            LOG.info("null binary entry: batchIndex " + batchIndex + " projection column num " + projectionColumnNum);
                        }
                        ((BytesWritable) convertSourceWritable).set(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesLength);
                        break;
                    case STRING:
                        if (deserializeRead.currentBytes == null) {
                            throw new RuntimeException("null string entry: batchIndex " + batchIndex + " projection column num " + projectionColumnNum);
                        }
                        // Use org.apache.hadoop.io.Text as our helper to go from byte[] to String.
                        ((Text) convertSourceWritable).set(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesLength);
                        break;
                    case VARCHAR:
                        {
                            // that does not use Java String objects.
                            if (deserializeRead.currentBytes == null) {
                                throw new RuntimeException("null varchar entry: batchIndex " + batchIndex + " projection column num " + projectionColumnNum);
                            }
                            int adjustedLength = StringExpr.truncate(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesLength, maxLengths[logicalColumnIndex]);
                            ((HiveVarcharWritable) convertSourceWritable).set(new String(deserializeRead.currentBytes, deserializeRead.currentBytesStart, adjustedLength, Charsets.UTF_8), -1);
                        }
                        break;
                    case CHAR:
                        {
                            // that does not use Java String objects.
                            if (deserializeRead.currentBytes == null) {
                                throw new RuntimeException("null char entry: batchIndex " + batchIndex + " projection column num " + projectionColumnNum);
                            }
                            int adjustedLength = StringExpr.rightTrimAndTruncate(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesLength, maxLengths[logicalColumnIndex]);
                            ((HiveCharWritable) convertSourceWritable).set(new String(deserializeRead.currentBytes, deserializeRead.currentBytesStart, adjustedLength, Charsets.UTF_8), -1);
                        }
                        break;
                    case DECIMAL:
                        ((HiveDecimalWritable) convertSourceWritable).set(deserializeRead.currentHiveDecimalWritable);
                        break;
                    case INTERVAL_YEAR_MONTH:
                        ((HiveIntervalYearMonthWritable) convertSourceWritable).set(deserializeRead.currentHiveIntervalYearMonthWritable);
                        break;
                    case INTERVAL_DAY_TIME:
                        ((HiveIntervalDayTimeWritable) convertSourceWritable).set(deserializeRead.currentHiveIntervalDayTimeWritable);
                        break;
                    default:
                        throw new RuntimeException("Primitive category " + sourcePrimitiveCategories[logicalColumnIndex] + " not supported");
                }
            }
            break;
        default:
            throw new RuntimeException("Category " + sourceCategories[logicalColumnIndex] + " not supported");
    }
    /*
     * Convert our source object we just read into the target object and store that in the
     * VectorizedRowBatch.
     */
    convertVectorAssignRow.assignConvertRowColumn(batch, batchIndex, logicalColumnIndex, convertSourceWritable);
}