Examples with DateWritableV2 org.apache.hadoop.hive.serde2.io.DateWritableV2 used on opensource projects

Example 6 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class TestHiveAccumuloTypes method testBinaryTypes.

@Test
public void testBinaryTypes() throws Exception {
    final String tableName = test.getMethodName(), user = "root", pass = "";
    MockInstance mockInstance = new MockInstance(test.getMethodName());
    Connector conn = mockInstance.getConnector(user, new PasswordToken(pass));
    HiveAccumuloTableInputFormat inputformat = new HiveAccumuloTableInputFormat();
    JobConf conf = new JobConf();
    conf.set(AccumuloSerDeParameters.TABLE_NAME, tableName);
    conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true");
    conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName());
    conf.set(AccumuloSerDeParameters.USER_NAME, user);
    conf.set(AccumuloSerDeParameters.USER_PASS, pass);
    // not used for mock, but
    conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181");
    // required by input format.
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, AccumuloHiveConstants.ROWID + ",cf:string,cf:boolean,cf:tinyint,cf:smallint,cf:int,cf:bigint" + ",cf:float,cf:double,cf:decimal,cf:date,cf:timestamp,cf:char,cf:varchar");
    conf.set(serdeConstants.LIST_COLUMNS, "string,string,boolean,tinyint,smallint,int,bigint,float,double,decimal,date,timestamp,char(4),varchar(7)");
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,string,boolean,tinyint,smallint,int,bigint,float,double,decimal,date,timestamp,char(4),varchar(7)");
    conf.set(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE, "binary");
    conn.tableOperations().create(tableName);
    BatchWriterConfig writerConf = new BatchWriterConfig();
    BatchWriter writer = conn.createBatchWriter(tableName, writerConf);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    String cf = "cf";
    byte[] cfBytes = cf.getBytes();
    Mutation m = new Mutation("row1");
    // string
    String stringValue = "string";
    JavaStringObjectInspector stringOI = (JavaStringObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME));
    LazyUtils.writePrimitiveUTF8(baos, stringOI.create(stringValue), stringOI, false, (byte) 0, null);
    m.put(cfBytes, "string".getBytes(), baos.toByteArray());
    // boolean
    boolean booleanValue = true;
    baos.reset();
    JavaBooleanObjectInspector booleanOI = (JavaBooleanObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
    LazyUtils.writePrimitive(baos, booleanOI.create(booleanValue), booleanOI);
    m.put(cfBytes, "boolean".getBytes(), baos.toByteArray());
    // tinyint
    byte tinyintValue = -127;
    baos.reset();
    JavaByteObjectInspector byteOI = (JavaByteObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
    LazyUtils.writePrimitive(baos, tinyintValue, byteOI);
    m.put(cfBytes, "tinyint".getBytes(), baos.toByteArray());
    // smallint
    short smallintValue = Short.MAX_VALUE;
    baos.reset();
    JavaShortObjectInspector shortOI = (JavaShortObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
    LazyUtils.writePrimitive(baos, smallintValue, shortOI);
    m.put(cfBytes, "smallint".getBytes(), baos.toByteArray());
    // int
    int intValue = Integer.MAX_VALUE;
    baos.reset();
    JavaIntObjectInspector intOI = (JavaIntObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
    LazyUtils.writePrimitive(baos, intValue, intOI);
    m.put(cfBytes, "int".getBytes(), baos.toByteArray());
    // bigint
    long bigintValue = Long.MAX_VALUE;
    baos.reset();
    JavaLongObjectInspector longOI = (JavaLongObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
    LazyUtils.writePrimitive(baos, bigintValue, longOI);
    m.put(cfBytes, "bigint".getBytes(), baos.toByteArray());
    // float
    float floatValue = Float.MAX_VALUE;
    baos.reset();
    JavaFloatObjectInspector floatOI = (JavaFloatObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
    LazyUtils.writePrimitive(baos, floatValue, floatOI);
    m.put(cfBytes, "float".getBytes(), baos.toByteArray());
    // double
    double doubleValue = Double.MAX_VALUE;
    baos.reset();
    JavaDoubleObjectInspector doubleOI = (JavaDoubleObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
    LazyUtils.writePrimitive(baos, doubleValue, doubleOI);
    m.put(cfBytes, "double".getBytes(), baos.toByteArray());
    // decimal
    baos.reset();
    HiveDecimal decimalValue = HiveDecimal.create(65536l);
    HiveDecimalWritable decimalWritable = new HiveDecimalWritable(decimalValue);
    decimalWritable.write(out);
    m.put(cfBytes, "decimal".getBytes(), baos.toByteArray());
    // date
    baos.reset();
    Date now = Date.ofEpochMilli(System.currentTimeMillis());
    DateWritableV2 dateWritable = new DateWritableV2(now);
    Date dateValue = dateWritable.get();
    dateWritable.write(out);
    m.put(cfBytes, "date".getBytes(), baos.toByteArray());
    // tiemestamp
    baos.reset();
    Timestamp timestampValue = Timestamp.ofEpochMilli(System.currentTimeMillis());
    ByteStream.Output output = new ByteStream.Output();
    TimestampWritableV2 timestampWritable = new TimestampWritableV2(timestampValue);
    timestampWritable.write(new DataOutputStream(output));
    output.close();
    m.put(cfBytes, "timestamp".getBytes(), output.toByteArray());
    // char
    baos.reset();
    HiveChar charValue = new HiveChar("char", 4);
    JavaHiveCharObjectInspector charOI = (JavaHiveCharObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(new CharTypeInfo(4));
    LazyUtils.writePrimitiveUTF8(baos, charOI.create(charValue), charOI, false, (byte) 0, null);
    m.put(cfBytes, "char".getBytes(), baos.toByteArray());
    baos.reset();
    HiveVarchar varcharValue = new HiveVarchar("varchar", 7);
    JavaHiveVarcharObjectInspector varcharOI = (JavaHiveVarcharObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(new VarcharTypeInfo(7));
    LazyUtils.writePrimitiveUTF8(baos, varcharOI.create(varcharValue), varcharOI, false, (byte) 0, null);
    m.put(cfBytes, "varchar".getBytes(), baos.toByteArray());
    writer.addMutation(m);
    writer.close();
    for (Entry<Key, Value> e : conn.createScanner(tableName, new Authorizations())) {
        System.out.println(e);
    }
    // Create the RecordReader
    FileInputFormat.addInputPath(conf, new Path("unused"));
    InputSplit[] splits = inputformat.getSplits(conf, 0);
    assertEquals(splits.length, 1);
    RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
    Text key = reader.createKey();
    AccumuloHiveRow value = reader.createValue();
    reader.next(key, value);
    Assert.assertEquals(13, value.getTuples().size());
    ByteArrayRef byteRef = new ByteArrayRef();
    // string
    Text cfText = new Text(cf), cqHolder = new Text();
    cqHolder.set("string");
    byte[] valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyStringObjectInspector lazyStringOI = LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, (byte) 0);
    LazyString lazyString = (LazyString) LazyFactory.createLazyObject(lazyStringOI);
    lazyString.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(stringValue, lazyString.getWritableObject().toString());
    // boolean
    cqHolder.set("boolean");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyBooleanObjectInspector lazyBooleanOI = (LazyBooleanObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
    LazyBoolean lazyBoolean = (LazyBoolean) LazyFactory.createLazyPrimitiveBinaryClass(lazyBooleanOI);
    lazyBoolean.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(booleanValue, lazyBoolean.getWritableObject().get());
    // tinyint
    cqHolder.set("tinyint");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyByteObjectInspector lazyByteOI = (LazyByteObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
    LazyByte lazyByte = (LazyByte) LazyFactory.createLazyPrimitiveBinaryClass(lazyByteOI);
    lazyByte.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(tinyintValue, lazyByte.getWritableObject().get());
    // smallint
    cqHolder.set("smallint");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyShortObjectInspector lazyShortOI = (LazyShortObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
    LazyShort lazyShort = (LazyShort) LazyFactory.createLazyPrimitiveBinaryClass(lazyShortOI);
    lazyShort.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(smallintValue, lazyShort.getWritableObject().get());
    // int
    cqHolder.set("int");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyIntObjectInspector lazyIntOI = (LazyIntObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
    LazyInteger lazyInt = (LazyInteger) LazyFactory.createLazyPrimitiveBinaryClass(lazyIntOI);
    lazyInt.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(intValue, lazyInt.getWritableObject().get());
    // bigint
    cqHolder.set("bigint");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyLongObjectInspector lazyLongOI = (LazyLongObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
    LazyLong lazyLong = (LazyLong) LazyFactory.createLazyPrimitiveBinaryClass(lazyLongOI);
    lazyLong.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(bigintValue, lazyLong.getWritableObject().get());
    // float
    cqHolder.set("float");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyFloatObjectInspector lazyFloatOI = (LazyFloatObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
    LazyFloat lazyFloat = (LazyFloat) LazyFactory.createLazyPrimitiveBinaryClass(lazyFloatOI);
    lazyFloat.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(floatValue, lazyFloat.getWritableObject().get(), 0);
    // double
    cqHolder.set("double");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyDoubleObjectInspector lazyDoubleOI = (LazyDoubleObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
    LazyDouble lazyDouble = (LazyDouble) LazyFactory.createLazyPrimitiveBinaryClass(lazyDoubleOI);
    lazyDouble.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(doubleValue, lazyDouble.getWritableObject().get(), 0);
    // decimal
    cqHolder.set("decimal");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes);
    DataInputStream in = new DataInputStream(bais);
    decimalWritable.readFields(in);
    Assert.assertEquals(decimalValue, decimalWritable.getHiveDecimal());
    // date
    cqHolder.set("date");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    bais = new ByteArrayInputStream(valueBytes);
    in = new DataInputStream(bais);
    dateWritable.readFields(in);
    Assert.assertEquals(dateValue, dateWritable.get());
    // timestamp
    cqHolder.set("timestamp");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    bais = new ByteArrayInputStream(valueBytes);
    in = new DataInputStream(bais);
    timestampWritable.readFields(in);
    Assert.assertEquals(timestampValue, timestampWritable.getTimestamp());
    // char
    cqHolder.set("char");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyHiveCharObjectInspector lazyCharOI = (LazyHiveCharObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new CharTypeInfo(4));
    LazyHiveChar lazyChar = (LazyHiveChar) LazyFactory.createLazyObject(lazyCharOI);
    lazyChar.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(charValue, lazyChar.getWritableObject().getHiveChar());
    // varchar
    cqHolder.set("varchar");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);
    byteRef.setData(valueBytes);
    LazyHiveVarcharObjectInspector lazyVarcharOI = (LazyHiveVarcharObjectInspector) LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(new VarcharTypeInfo(7));
    LazyHiveVarchar lazyVarchar = (LazyHiveVarchar) LazyFactory.createLazyObject(lazyVarcharOI);
    lazyVarchar.init(byteRef, 0, valueBytes.length);
    Assert.assertEquals(varcharValue.toString(), lazyVarchar.getWritableObject().getHiveVarchar().toString());
}

Example 7 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class ReaderWriter method writeDatum.

public static void writeDatum(DataOutput out, Object val) throws IOException {
    // write the data type
    byte type = DataType.findType(val);
    out.write(type);
    switch(type) {
        case DataType.LIST:
            List<?> list = (List<?>) val;
            int sz = list.size();
            out.writeInt(sz);
            for (int i = 0; i < sz; i++) {
                writeDatum(out, list.get(i));
            }
            return;
        case DataType.MAP:
            Map<?, ?> m = (Map<?, ?>) val;
            out.writeInt(m.size());
            Iterator<?> i = m.entrySet().iterator();
            while (i.hasNext()) {
                Entry<?, ?> entry = (Entry<?, ?>) i.next();
                writeDatum(out, entry.getKey());
                writeDatum(out, entry.getValue());
            }
            return;
        case DataType.INTEGER:
            new VIntWritable((Integer) val).write(out);
            return;
        case DataType.LONG:
            new VLongWritable((Long) val).write(out);
            return;
        case DataType.FLOAT:
            out.writeFloat((Float) val);
            return;
        case DataType.DOUBLE:
            out.writeDouble((Double) val);
            return;
        case DataType.BOOLEAN:
            out.writeBoolean((Boolean) val);
            return;
        case DataType.BYTE:
            out.writeByte((Byte) val);
            return;
        case DataType.SHORT:
            out.writeShort((Short) val);
            return;
        case DataType.STRING:
            String s = (String) val;
            byte[] utfBytes = s.getBytes(ReaderWriter.UTF8);
            out.writeInt(utfBytes.length);
            out.write(utfBytes);
            return;
        case DataType.BINARY:
            byte[] ba = (byte[]) val;
            out.writeInt(ba.length);
            out.write(ba);
            return;
        case DataType.NULL:
            // for NULL we just write out the type
            return;
        case DataType.CHAR:
            new HiveCharWritable((HiveChar) val).write(out);
            return;
        case DataType.VARCHAR:
            new HiveVarcharWritable((HiveVarchar) val).write(out);
            return;
        case DataType.DECIMAL:
            new HiveDecimalWritable((HiveDecimal) val).write(out);
            return;
        case DataType.DATE:
            new DateWritableV2((Date) val).write(out);
            return;
        case DataType.TIMESTAMP:
            new TimestampWritableV2((Timestamp) val).write(out);
            return;
        default:
            throw new IOException("Unexpected data type " + type + " found in stream.");
    }
}

Example 8 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class VectorizedBatchUtil method setVector.

private static void setVector(Object row, StructObjectInspector oi, StructField field, VectorizedRowBatch batch, DataOutputBuffer buffer, int rowIndex, int colIndex, int offset) throws HiveException {
    Object fieldData = oi.getStructFieldData(row, field);
    ObjectInspector foi = field.getFieldObjectInspector();
    // Vectorization only supports PRIMITIVE data types. Assert the same
    assert (foi.getCategory() == Category.PRIMITIVE);
    // Get writable object
    PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi;
    Object writableCol = poi.getPrimitiveWritableObject(fieldData);
    // float/double. String types have no default value for null.
    switch(poi.getPrimitiveCategory()) {
        case BOOLEAN:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.vector[rowIndex] = ((BooleanWritable) writableCol).get() ? 1 : 0;
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case BYTE:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.vector[rowIndex] = ((ByteWritable) writableCol).get();
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case SHORT:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.vector[rowIndex] = ((ShortWritable) writableCol).get();
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case INT:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.vector[rowIndex] = ((IntWritable) writableCol).get();
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case LONG:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.vector[rowIndex] = ((LongWritable) writableCol).get();
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case DATE:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.vector[rowIndex] = ((DateWritableV2) writableCol).getDays();
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case FLOAT:
            {
                DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    dcv.vector[rowIndex] = ((FloatWritable) writableCol).get();
                    dcv.isNull[rowIndex] = false;
                } else {
                    dcv.vector[rowIndex] = Double.NaN;
                    setNullColIsNullValue(dcv, rowIndex);
                }
            }
            break;
        case DOUBLE:
            {
                DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    dcv.vector[rowIndex] = ((DoubleWritable) writableCol).get();
                    dcv.isNull[rowIndex] = false;
                } else {
                    dcv.vector[rowIndex] = Double.NaN;
                    setNullColIsNullValue(dcv, rowIndex);
                }
            }
            break;
        case TIMESTAMP:
            {
                TimestampColumnVector lcv = (TimestampColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    lcv.set(rowIndex, ((TimestampWritableV2) writableCol).getTimestamp().toSqlTimestamp());
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.setNullValue(rowIndex);
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case INTERVAL_YEAR_MONTH:
            {
                LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    HiveIntervalYearMonth i = ((HiveIntervalYearMonthWritable) writableCol).getHiveIntervalYearMonth();
                    lcv.vector[rowIndex] = i.getTotalMonths();
                    lcv.isNull[rowIndex] = false;
                } else {
                    lcv.vector[rowIndex] = 1;
                    setNullColIsNullValue(lcv, rowIndex);
                }
            }
            break;
        case INTERVAL_DAY_TIME:
            {
                IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    HiveIntervalDayTime idt = ((HiveIntervalDayTimeWritable) writableCol).getHiveIntervalDayTime();
                    icv.set(rowIndex, idt);
                    icv.isNull[rowIndex] = false;
                } else {
                    icv.setNullValue(rowIndex);
                    setNullColIsNullValue(icv, rowIndex);
                }
            }
            break;
        case BINARY:
            {
                BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    bcv.isNull[rowIndex] = false;
                    BytesWritable bw = (BytesWritable) writableCol;
                    byte[] bytes = bw.getBytes();
                    int start = buffer.getLength();
                    int length = bw.getLength();
                    try {
                        buffer.write(bytes, 0, length);
                    } catch (IOException ioe) {
                        throw new IllegalStateException("bad write", ioe);
                    }
                    bcv.setRef(rowIndex, buffer.getData(), start, length);
                } else {
                    setNullColIsNullValue(bcv, rowIndex);
                }
            }
            break;
        case STRING:
            {
                BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    bcv.isNull[rowIndex] = false;
                    Text colText = (Text) writableCol;
                    int start = buffer.getLength();
                    int length = colText.getLength();
                    try {
                        buffer.write(colText.getBytes(), 0, length);
                    } catch (IOException ioe) {
                        throw new IllegalStateException("bad write", ioe);
                    }
                    bcv.setRef(rowIndex, buffer.getData(), start, length);
                } else {
                    setNullColIsNullValue(bcv, rowIndex);
                }
            }
            break;
        case CHAR:
            {
                BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    bcv.isNull[rowIndex] = false;
                    HiveChar colHiveChar = ((HiveCharWritable) writableCol).getHiveChar();
                    byte[] bytes = colHiveChar.getStrippedValue().getBytes();
                    // We assume the CHAR maximum length was enforced when the object was created.
                    int length = bytes.length;
                    int start = buffer.getLength();
                    try {
                        // In vector mode, we store CHAR as unpadded.
                        buffer.write(bytes, 0, length);
                    } catch (IOException ioe) {
                        throw new IllegalStateException("bad write", ioe);
                    }
                    bcv.setRef(rowIndex, buffer.getData(), start, length);
                } else {
                    setNullColIsNullValue(bcv, rowIndex);
                }
            }
            break;
        case VARCHAR:
            {
                BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex];
                if (writableCol != null) {
                    bcv.isNull[rowIndex] = false;
                    HiveVarchar colHiveVarchar = ((HiveVarcharWritable) writableCol).getHiveVarchar();
                    byte[] bytes = colHiveVarchar.getValue().getBytes();
                    // We assume the VARCHAR maximum length was enforced when the object was created.
                    int length = bytes.length;
                    int start = buffer.getLength();
                    try {
                        buffer.write(bytes, 0, length);
                    } catch (IOException ioe) {
                        throw new IllegalStateException("bad write", ioe);
                    }
                    bcv.setRef(rowIndex, buffer.getData(), start, length);
                } else {
                    setNullColIsNullValue(bcv, rowIndex);
                }
            }
            break;
        case DECIMAL:
            DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[offset + colIndex];
            if (writableCol != null) {
                dcv.isNull[rowIndex] = false;
                HiveDecimalWritable wobj = (HiveDecimalWritable) writableCol;
                dcv.set(rowIndex, wobj);
            } else {
                setNullColIsNullValue(dcv, rowIndex);
            }
            break;
        default:
            throw new HiveException("Vectorizaton is not supported for datatype:" + poi.getPrimitiveCategory());
    }
}

Example 9 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class VectorizedRowBatchCtx method addPartitionColsToBatch.

public void addPartitionColsToBatch(ColumnVector col, Object value, int colIndex) {
    String partitionColumnName = rowColumnNames[colIndex];
    PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) rowColumnTypeInfos[colIndex];
    switch(primitiveTypeInfo.getPrimitiveCategory()) {
        case BOOLEAN:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill((Boolean) value == true ? 1 : 0);
                }
            }
            break;
        case BYTE:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill((Byte) value);
                }
            }
            break;
        case SHORT:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill((Short) value);
                }
            }
            break;
        case INT:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill((Integer) value);
                }
            }
            break;
        case LONG:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill((Long) value);
                }
            }
            break;
        case DATE:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill(DateWritableV2.dateToDays((Date) value));
                }
            }
            break;
        case TIMESTAMP:
            {
                TimestampColumnVector lcv = (TimestampColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill(((Timestamp) value).toSqlTimestamp());
                }
            }
            break;
        case INTERVAL_YEAR_MONTH:
            {
                LongColumnVector lcv = (LongColumnVector) col;
                if (value == null) {
                    lcv.noNulls = false;
                    lcv.isNull[0] = true;
                    lcv.isRepeating = true;
                } else {
                    lcv.fill(((HiveIntervalYearMonth) value).getTotalMonths());
                }
            }
            break;
        case INTERVAL_DAY_TIME:
            {
                IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) col;
                if (value == null) {
                    icv.noNulls = false;
                    icv.isNull[0] = true;
                    icv.isRepeating = true;
                } else {
                    icv.fill(((HiveIntervalDayTime) value));
                }
            }
            break;
        case FLOAT:
            {
                DoubleColumnVector dcv = (DoubleColumnVector) col;
                if (value == null) {
                    dcv.noNulls = false;
                    dcv.isNull[0] = true;
                    dcv.isRepeating = true;
                } else {
                    dcv.fill((Float) value);
                }
            }
            break;
        case DOUBLE:
            {
                DoubleColumnVector dcv = (DoubleColumnVector) col;
                if (value == null) {
                    dcv.noNulls = false;
                    dcv.isNull[0] = true;
                    dcv.isRepeating = true;
                } else {
                    dcv.fill((Double) value);
                }
            }
            break;
        case DECIMAL:
            {
                DataTypePhysicalVariation dataTypePhysicalVariation = rowDataTypePhysicalVariations != null ? rowDataTypePhysicalVariations[colIndex] : DataTypePhysicalVariation.NONE;
                if (dataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
                    Decimal64ColumnVector dv = (Decimal64ColumnVector) col;
                    if (value == null) {
                        dv.noNulls = false;
                        dv.isNull[0] = true;
                        dv.isRepeating = true;
                    } else {
                        dv.fill(((HiveDecimal) value).longValue());
                    }
                } else {
                    DecimalColumnVector dv = (DecimalColumnVector) col;
                    if (value == null) {
                        dv.noNulls = false;
                        dv.isNull[0] = true;
                        dv.isRepeating = true;
                    } else {
                        dv.fill((HiveDecimal) value);
                    }
                }
            }
            break;
        case BINARY:
            {
                BytesColumnVector bcv = (BytesColumnVector) col;
                byte[] bytes = (byte[]) value;
                if (bytes == null) {
                    bcv.noNulls = false;
                    bcv.isNull[0] = true;
                    bcv.isRepeating = true;
                } else {
                    bcv.fill(bytes);
                }
            }
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            {
                BytesColumnVector bcv = (BytesColumnVector) col;
                String sVal = value.toString();
                if (sVal == null) {
                    bcv.noNulls = false;
                    bcv.isNull[0] = true;
                    bcv.isRepeating = true;
                } else {
                    bcv.fill(sVal.getBytes());
                }
            }
            break;
        default:
            throw new RuntimeException("Unable to recognize the partition type " + primitiveTypeInfo.getPrimitiveCategory() + " for column " + partitionColumnName);
    }
}

Example 10 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class VectorUDFDateDiffColScalar method evaluate.

@Override
public void evaluate(VectorizedRowBatch batch) throws HiveException {
    if (childExpressions != null) {
        super.evaluateChildren(batch);
    }
    LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum];
    ColumnVector inputCol = batch.cols[this.inputColumnNum[0]];
    /* every line below this is identical for evaluateLong & evaluateString */
    final int n = inputCol.isRepeating ? 1 : batch.size;
    int[] sel = batch.selected;
    final boolean selectedInUse = (inputCol.isRepeating == false) && batch.selectedInUse;
    boolean[] outputIsNull = outputColVector.isNull;
    if (batch.size == 0) {
        /* n != batch.size when isRepeating */
        return;
    }
    // We do not need to do a column reset since we are carefully changing the output.
    outputColVector.isRepeating = false;
    PrimitiveCategory primitiveCategory1 = ((PrimitiveTypeInfo) inputTypeInfos[1]).getPrimitiveCategory();
    switch(primitiveCategory1) {
        case DATE:
            baseDate = (int) longValue;
            break;
        case TIMESTAMP:
            date.setTime(timestampValue.getTime());
            baseDate = DateWritableV2.dateToDays(date);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            try {
                org.apache.hadoop.hive.common.type.Date hiveDate = org.apache.hadoop.hive.common.type.Date.valueOf(new String(bytesValue, StandardCharsets.UTF_8));
                baseDate = hiveDate.toEpochDay();
                break;
            } catch (Exception e) {
                outputColVector.noNulls = false;
                if (selectedInUse) {
                    for (int j = 0; j < n; j++) {
                        int i = sel[j];
                        outputColVector.isNull[i] = true;
                    }
                } else {
                    for (int i = 0; i < n; i++) {
                        outputColVector.isNull[i] = true;
                    }
                }
                return;
            }
        default:
            throw new Error("Invalid input type #1: " + primitiveCategory1.name());
    }
    PrimitiveCategory primitiveCategory0 = ((PrimitiveTypeInfo) inputTypeInfos[0]).getPrimitiveCategory();
    switch(primitiveCategory0) {
        case DATE:
            if (inputCol.isRepeating) {
                if (inputCol.noNulls || !inputCol.isNull[0]) {
                    outputColVector.isNull[0] = false;
                    outputColVector.vector[0] = evaluateDate(inputCol, 0);
                } else {
                    outputColVector.isNull[0] = true;
                    outputColVector.noNulls = false;
                }
                outputColVector.isRepeating = true;
            } else if (inputCol.noNulls) {
                if (batch.selectedInUse) {
                    if (!outputColVector.noNulls) {
                        for (int j = 0; j != n; j++) {
                            final int i = sel[j];
                            // Set isNull before call in case it changes it mind.
                            outputIsNull[i] = false;
                            outputColVector.vector[i] = evaluateDate(inputCol, i);
                        }
                    } else {
                        for (int j = 0; j != n; j++) {
                            final int i = sel[j];
                            outputColVector.vector[i] = evaluateDate(inputCol, i);
                        }
                    }
                } else {
                    if (!outputColVector.noNulls) {
                        // Assume it is almost always a performance win to fill all of isNull so we can
                        // safely reset noNulls.
                        Arrays.fill(outputIsNull, false);
                        outputColVector.noNulls = true;
                    }
                    for (int i = 0; i != n; i++) {
                        outputColVector.vector[i] = evaluateDate(inputCol, i);
                    }
                }
            } else /* there are nulls in the inputColVector */
            {
                // Carefully handle NULLs..
                // Handle case with nulls. Don't do function if the value is null, to save time,
                // because calling the function can be expensive.
                outputColVector.noNulls = false;
                if (selectedInUse) {
                    for (int j = 0; j < n; j++) {
                        int i = sel[j];
                        outputColVector.isNull[i] = inputCol.isNull[i];
                        if (!inputCol.isNull[i]) {
                            outputColVector.vector[i] = evaluateDate(inputCol, i);
                        }
                    }
                } else {
                    for (int i = 0; i < n; i++) {
                        outputColVector.isNull[i] = inputCol.isNull[i];
                        if (!inputCol.isNull[i]) {
                            outputColVector.vector[i] = evaluateDate(inputCol, i);
                        }
                    }
                }
            }
            break;
        case TIMESTAMP:
            if (inputCol.isRepeating) {
                if (inputCol.noNulls || !inputCol.isNull[0]) {
                    outputColVector.isNull[0] = false;
                    outputColVector.vector[0] = evaluateTimestamp(inputCol, 0);
                } else {
                    outputColVector.isNull[0] = true;
                    outputColVector.noNulls = false;
                }
                outputColVector.isRepeating = true;
            } else if (inputCol.noNulls) {
                if (batch.selectedInUse) {
                    if (!outputColVector.noNulls) {
                        for (int j = 0; j != n; j++) {
                            final int i = sel[j];
                            // Set isNull before call in case it changes it mind.
                            outputIsNull[i] = false;
                            outputColVector.vector[i] = evaluateTimestamp(inputCol, i);
                        }
                    } else {
                        for (int j = 0; j != n; j++) {
                            final int i = sel[j];
                            outputColVector.vector[i] = evaluateTimestamp(inputCol, i);
                        }
                    }
                } else {
                    if (!outputColVector.noNulls) {
                        // Assume it is almost always a performance win to fill all of isNull so we can
                        // safely reset noNulls.
                        Arrays.fill(outputIsNull, false);
                        outputColVector.noNulls = true;
                    }
                    for (int i = 0; i != n; i++) {
                        outputColVector.vector[i] = evaluateTimestamp(inputCol, i);
                    }
                }
            } else /* there are nulls in the inputColVector */
            {
                // Carefully handle NULLs..
                // Handle case with nulls. Don't do function if the value is null, to save time,
                // because calling the function can be expensive.
                outputColVector.noNulls = false;
                if (selectedInUse) {
                    for (int j = 0; j < n; j++) {
                        int i = sel[j];
                        outputColVector.isNull[i] = inputCol.isNull[i];
                        if (!inputCol.isNull[i]) {
                            outputColVector.vector[i] = evaluateTimestamp(inputCol, i);
                        }
                    }
                } else {
                    for (int i = 0; i < n; i++) {
                        outputColVector.isNull[i] = inputCol.isNull[i];
                        if (!inputCol.isNull[i]) {
                            outputColVector.vector[i] = evaluateTimestamp(inputCol, i);
                        }
                    }
                }
            }
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            if (inputCol.isRepeating) {
                if (inputCol.noNulls || !inputCol.isNull[0]) {
                    outputColVector.isNull[0] = false;
                    evaluateString(inputCol, outputColVector, 0);
                } else {
                    outputColVector.isNull[0] = true;
                    outputColVector.noNulls = false;
                }
                outputColVector.isRepeating = true;
            } else if (inputCol.noNulls) {
                if (batch.selectedInUse) {
                    if (!outputColVector.noNulls) {
                        for (int j = 0; j != n; j++) {
                            final int i = sel[j];
                            // Set isNull before call in case it changes it mind.
                            outputIsNull[i] = false;
                            evaluateString(inputCol, outputColVector, i);
                        }
                    } else {
                        for (int j = 0; j != n; j++) {
                            final int i = sel[j];
                            evaluateString(inputCol, outputColVector, i);
                        }
                    }
                } else {
                    if (!outputColVector.noNulls) {
                        // Assume it is almost always a performance win to fill all of isNull so we can
                        // safely reset noNulls.
                        Arrays.fill(outputIsNull, false);
                        outputColVector.noNulls = true;
                    }
                    for (int i = 0; i != n; i++) {
                        evaluateString(inputCol, outputColVector, i);
                    }
                }
            } else /* there are nulls in the inputColVector */
            {
                // Carefully handle NULLs..
                // Handle case with nulls. Don't do function if the value is null, to save time,
                // because calling the function can be expensive.
                outputColVector.noNulls = false;
                if (selectedInUse) {
                    for (int j = 0; j < n; j++) {
                        int i = sel[j];
                        outputColVector.isNull[i] = inputCol.isNull[i];
                        if (!inputCol.isNull[i]) {
                            evaluateString(inputCol, outputColVector, i);
                        }
                    }
                } else {
                    for (int i = 0; i < n; i++) {
                        outputColVector.isNull[i] = inputCol.isNull[i];
                        if (!inputCol.isNull[i]) {
                            evaluateString(inputCol, outputColVector, i);
                        }
                    }
                }
            }
            break;
        default:
            throw new Error("Invalid input type #0: " + primitiveCategory0.name());
    }
}